@budibase/server 1.3.15-alpha.0 → 1.3.15-alpha.1

package/dist/utilities/fileSystem/index.js

@@ -14,7 +14,7 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
@@ -295,32 +295,47 @@ exports.cleanup = appIds => {
         }
     }
 };
-exports.extractPluginTarball = (file) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!file.name.endsWith(".tar.gz")) {
-        throw new Error("Plugin must be compressed into a gzipped tarball.");
+const createTempFolder = item => {
+    const path = join(budibaseTempDir(), item);
+    try {
+        // remove old tmp directories automatically - don't combine
+        if (fs.existsSync(path)) {
+            fs.rmSync(path, { recursive: true, force: true });
+        }
+        fs.mkdirSync(path);
     }
-    const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0]);
-    // remove old tmp directories automatically - don't combine
-    if (fs.existsSync(path)) {
-        fs.rmSync(path, { recursive: true, force: true });
+    catch (err) {
+        throw new Error(`Path cannot be created: ${err.message}`);
     }
-    fs.mkdirSync(path);
+    return path;
+};
+exports.createTempFolder = createTempFolder;
+const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
     yield tar.extract({
-        file: file.path,
-        C: path,
+        file: fromFilePath,
+        C: toPath,
     });
+});
+exports.extractTarball = extractTarball;
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
     let metadata = {};
     try {
         const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
         const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
         metadata.schema = JSON.parse(schema);
         metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
+        }
     }
     catch (err) {
-        throw new Error("Unable to process schema.json/package.json in plugin.");
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
     }
     return { metadata, directory: path };
 });
+exports.getPluginMetadata = getPluginMetadata;
 exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
     if (!fs.existsSync(DATASOURCE_PATH)) {
         fs.mkdirSync(DATASOURCE_PATH);
@@ -350,6 +365,34 @@ exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, voi
         throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
     }
 });
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+    if (!fs.existsSync(startPath)) {
+        return;
+    }
+    const files = fs.readdirSync(startPath);
+    for (let i = 0, len = files.length; i < len; i++) {
+        const filename = join(startPath, files[i]);
+        const stat = fs.lstatSync(filename);
+        if (stat.isDirectory()) {
+            return exports.findFileRec(filename, filter);
+        }
+        else if (filename.endsWith(filter)) {
+            return filename;
+        }
+    }
+};
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+    if (!fs.existsSync(path)) {
+        return;
+    }
+    fs.rmSync(path, { recursive: true, force: true });
+};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -357,5 +400,6 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
+exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.15-alpha.0",
+  "version": "1.3.15-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.3.15-alpha.0",
-    "@budibase/client": "1.3.15-alpha.0",
-    "@budibase/pro": "1.3.13",
-    "@budibase/string-templates": "1.3.15-alpha.0",
-    "@budibase/types": "1.3.15-alpha.0",
+    "@budibase/backend-core": "1.3.15-alpha.1",
+    "@budibase/client": "1.3.15-alpha.1",
+    "@budibase/pro": "1.3.15-alpha.0",
+    "@budibase/string-templates": "1.3.15-alpha.1",
+    "@budibase/types": "1.3.15-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -198,5 +198,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "38f3c6d1d713f5e1bd18fdd81bdffb9bf7d4f9d3"
+  "gitHead": "a92a8ed882cde62db5903d495112e202173defc7"
 }

package/src/api/controllers/plugin/file.ts

@@ -0,0 +1,15 @@
+import {
+  createTempFolder,
+  getPluginMetadata,
+  extractTarball,
+} from "../../../utilities/fileSystem"
+
+export async function fileUpload(file: { name: string; path: string }) {
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/github.ts

@@ -0,0 +1,75 @@
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { downloadUnzipTarball } from "./utils"
+
+export async function request(
+  url: string,
+  headers: { [key: string]: string },
+  err: string
+) {
+  const response = await fetch(url, { headers })
+  if (response.status >= 300) {
+    const respErr = await response.text()
+    throw new Error(`Error: ${err} - ${respErr}`)
+  }
+  return response.json()
+}
+
+export async function githubUpload(url: string, name = "", token = "") {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
+  const pluginDetails = await request(
+    githubApiUrl,
+    headers,
+    "Repository not found"
+  )
+  const pluginName = pluginDetails.name || name
+  const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+    ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+    : undefined
+  if (!pluginLatestReleaseUrl) {
+    throw new Error("Github release not found")
+  }
+
+  const pluginReleaseDetails = await request(
+    pluginLatestReleaseUrl,
+    headers,
+    "Github latest release not found"
+  )
+  const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+    (x: any) => x?.["content_type"] === "application/gzip"
+  )
+  const pluginLastReleaseTarballUrl =
+    pluginReleaseTarballAsset?.["browser_download_url"]
+  if (!pluginLastReleaseTarballUrl) {
+    throw new Error("Github latest release url not found")
+  }
+
+  try {
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+    return await getPluginMetadata(path)
+  } catch (err: any) {
+    let errMsg = err?.message || err
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarball not found"
+    }
+    throw new Error(errMsg)
+  }
+}

package/src/api/controllers/plugin/index.ts

@@ -0,0 +1,202 @@
+import { ObjectStoreBuckets } from "../../../constants"
+import { loadJSFile } from "../../../utilities/fileSystem"
+import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
+import { validate } from "@budibase/backend-core/plugins"
+import { generatePluginID, getPluginParams } from "../../../db/utils"
+import {
+  uploadDirectory,
+  deleteFolder,
+} from "@budibase/backend-core/objectStore"
+import { PluginType, FileType, PluginSource } from "@budibase/types"
+import env from "../../../environment"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function upload(ctx: any) {
+  const plugins: FileType[] =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
+  try {
+    let docs = []
+    // can do single or multiple plugins
+    for (let plugin of plugins) {
+      const doc = await processPlugin(plugin, PluginSource.FILE)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function create(ctx: any) {
+  const { source, url, headers, githubToken } = ctx.request.body
+
+  if (!env.SELF_HOSTED) {
+    ctx.throw(400, "Plugins not supported outside of self-host.")
+  }
+
+  try {
+    let metadata
+    let directory
+    // Generating random name as a backup and needed for url
+    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+
+    switch (source) {
+      case PluginSource.NPM:
+        const { metadata: metadataNpm, directory: directoryNpm } =
+          await npmUpload(url, name)
+        metadata = metadataNpm
+        directory = directoryNpm
+        break
+      case PluginSource.GITHUB:
+        const { metadata: metadataGithub, directory: directoryGithub } =
+          await githubUpload(url, name, githubToken)
+        metadata = metadataGithub
+        directory = directoryGithub
+        break
+      case PluginSource.URL:
+        const headersObj = headers || {}
+        const { metadata: metadataUrl, directory: directoryUrl } =
+          await urlUpload(url, name, headersObj)
+        metadata = metadataUrl
+        directory = directoryUrl
+        break
+    }
+
+    validate(metadata?.schema)
+
+    const doc = await storePlugin(metadata, directory, source)
+
+    ctx.body = {
+      message: "Plugin uploaded successfully",
+      plugins: [doc],
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+  ctx.status = 200
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {
+  const db = getGlobalDB()
+  const { pluginId } = ctx.params
+
+  try {
+    const plugin = await db.get(pluginId)
+    const bucketPath = `${plugin.name}/`
+    await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath)
+
+    await db.remove(pluginId, plugin._rev)
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to delete plugin: ${errMsg}`)
+  }
+
+  ctx.message = `Plugin ${ctx.params.pluginId} deleted.`
+  ctx.status = 200
+}
+
+export async function storePlugin(
+  metadata: any,
+  directory: any,
+  source?: string
+) {
+  const db = getGlobalDB()
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  let doc = {
+    _id: pluginId,
+    _rev: rev,
+    ...metadata,
+    name,
+    version,
+    description,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+
+  if (source) {
+    doc = {
+      ...doc,
+      source,
+    }
+  }
+
+  const response = await db.put(doc)
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}
+
+export async function processPlugin(plugin: FileType, source?: string) {
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
+
+  const { metadata, directory } = await fileUpload(plugin)
+  return await storePlugin(metadata, directory, source)
+}

package/src/api/controllers/plugin/npm.ts

@@ -0,0 +1,56 @@
+import {
+  getPluginMetadata,
+  findFileRec,
+  extractTarball,
+  deleteFolderFileSystem,
+} from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { join } from "path"
+import { downloadUnzipTarball } from "./utils"
+
+export async function npmUpload(url: string, name: string, headers = {}) {
+  let npmTarballUrl = url
+  let pluginName = name
+
+  if (
+    !npmTarballUrl.includes("https://www.npmjs.com") &&
+    !npmTarballUrl.includes("https://registry.npmjs.org")
+  ) {
+    throw new Error("The plugin origin must be from NPM")
+  }
+
+  if (!npmTarballUrl.includes(".tgz")) {
+    const npmPackageURl = url.replace(
+      "https://www.npmjs.com/package/",
+      "https://registry.npmjs.org/"
+    )
+    const response = await fetch(npmPackageURl)
+    if (response.status !== 200) {
+      throw new Error("NPM Package not found")
+    }
+
+    let npmDetails = await response.json()
+    pluginName = npmDetails.name
+    const npmVersion = npmDetails["dist-tags"].latest
+    npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
+
+    if (!npmTarballUrl) {
+      throw new Error("NPM tarball url not found")
+    }
+  }
+
+  const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
+  const tarballPluginFile = findFileRec(path, ".tar.gz")
+  if (!tarballPluginFile) {
+    throw new Error("Tarball plugin file not found")
+  }
+
+  try {
+    await extractTarball(tarballPluginFile, path)
+    deleteFolderFileSystem(join(path, "package"))
+  } catch (err: any) {
+    throw new Error(err)
+  }
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/uploaders.ts

@@ -0,0 +1,4 @@
+export { fileUpload } from "./file"
+export { githubUpload } from "./github"
+export { npmUpload } from "./npm"
+export { urlUpload } from "./url"

package/src/api/controllers/plugin/url.ts

@@ -0,0 +1,12 @@
+import { downloadUnzipTarball } from "./utils"
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+
+export async function urlUpload(url: string, name = "", headers = {}) {
+  if (!url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  const path = await downloadUnzipTarball(url, name, headers)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/utils.ts

@@ -0,0 +1,19 @@
+import {
+  createTempFolder,
+  downloadTarballDirect,
+} from "../../../utilities/fileSystem"
+
+export async function downloadUnzipTarball(
+  url: string,
+  name: string,
+  headers = {}
+) {
+  try {
+    const path = createTempFolder(name)
+    await downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e: any) {
+    throw new Error(e.message)
+  }
+}

package/src/api/routes/plugin.ts

@@ -7,6 +7,7 @@ const router = new Router()
 
 router
   .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
+  .post("/api/plugin", authorized(BUILDER), controller.create)
   .get("/api/plugin", authorized(BUILDER), controller.fetch)
   .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
 

package/src/module.d.ts

@@ -11,3 +11,4 @@ declare module "@budibase/backend-core/encryption"
 declare module "@budibase/backend-core/utils"
 declare module "@budibase/backend-core/redis"
 declare module "@budibase/backend-core/objectStore"
+declare module "@budibase/backend-core/plugins"

package/src/utilities/fileSystem/index.js

@@ -15,6 +15,7 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  downloadTarballDirect,
   deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
@@ -338,31 +339,57 @@ exports.cleanup = appIds => {
   }
 }
 
-exports.extractPluginTarball = async file => {
-  if (!file.name.endsWith(".tar.gz")) {
-    throw new Error("Plugin must be compressed into a gzipped tarball.")
-  }
-  const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0])
-  // remove old tmp directories automatically - don't combine
-  if (fs.existsSync(path)) {
-    fs.rmSync(path, { recursive: true, force: true })
+const createTempFolder = item => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err) {
+    throw new Error(`Path cannot be created: ${err.message}`)
   }
-  fs.mkdirSync(path)
+
+  return path
+}
+exports.createTempFolder = createTempFolder
+
+const extractTarball = async (fromFilePath, toPath) => {
   await tar.extract({
-    file: file.path,
-    C: path,
+    file: fromFilePath,
+    C: toPath,
   })
+}
+exports.extractTarball = extractTarball
+
+const getPluginMetadata = async path => {
   let metadata = {}
   try {
     const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
     const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
     metadata.schema = JSON.parse(schema)
     metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
   } catch (err) {
-    throw new Error("Unable to process schema.json/package.json in plugin.")
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
   }
+
   return { metadata, directory: path }
 }
+exports.getPluginMetadata = getPluginMetadata
 
 exports.getDatasourcePlugin = async (name, url, hash) => {
   if (!fs.existsSync(DATASOURCE_PATH)) {
@@ -396,6 +423,38 @@ exports.getDatasourcePlugin = async (name, url, hash) => {
   }
 }
 
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}
+
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -403,5 +462,6 @@ exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
 exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH