@budibase/server 1.3.15-alpha.0 → 1.3.15-alpha.3

package/dist/utilities/fileSystem/index.js

@@ -14,7 +14,7 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
@@ -295,32 +295,47 @@ exports.cleanup = appIds => {
         }
     }
 };
-exports.extractPluginTarball = (file) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!file.name.endsWith(".tar.gz")) {
-        throw new Error("Plugin must be compressed into a gzipped tarball.");
+const createTempFolder = item => {
+    const path = join(budibaseTempDir(), item);
+    try {
+        // remove old tmp directories automatically - don't combine
+        if (fs.existsSync(path)) {
+            fs.rmSync(path, { recursive: true, force: true });
+        }
+        fs.mkdirSync(path);
     }
-    const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0]);
-    // remove old tmp directories automatically - don't combine
-    if (fs.existsSync(path)) {
-        fs.rmSync(path, { recursive: true, force: true });
+    catch (err) {
+        throw new Error(`Path cannot be created: ${err.message}`);
     }
-    fs.mkdirSync(path);
+    return path;
+};
+exports.createTempFolder = createTempFolder;
+const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
     yield tar.extract({
-        file: file.path,
-        C: path,
+        file: fromFilePath,
+        C: toPath,
     });
+});
+exports.extractTarball = extractTarball;
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
     let metadata = {};
     try {
         const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
         const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
         metadata.schema = JSON.parse(schema);
         metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
+        }
     }
     catch (err) {
-        throw new Error("Unable to process schema.json/package.json in plugin.");
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
     }
     return { metadata, directory: path };
 });
+exports.getPluginMetadata = getPluginMetadata;
 exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
     if (!fs.existsSync(DATASOURCE_PATH)) {
         fs.mkdirSync(DATASOURCE_PATH);
@@ -350,6 +365,34 @@ exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, voi
         throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
     }
 });
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+    if (!fs.existsSync(startPath)) {
+        return;
+    }
+    const files = fs.readdirSync(startPath);
+    for (let i = 0, len = files.length; i < len; i++) {
+        const filename = join(startPath, files[i]);
+        const stat = fs.lstatSync(filename);
+        if (stat.isDirectory()) {
+            return exports.findFileRec(filename, filter);
+        }
+        else if (filename.endsWith(filter)) {
+            return filename;
+        }
+    }
+};
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+    if (!fs.existsSync(path)) {
+        return;
+    }
+    fs.rmSync(path, { recursive: true, force: true });
+};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -357,5 +400,6 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
+exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/dist/websocket.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClientAppSocket = exports.initialise = void 0;
+const socket_io_1 = require("socket.io");
+class Websocket {
+    constructor(server, path) {
+        this.socketServer = new socket_io_1.Server(server, {
+            path,
+        });
+    }
+    // Emit an event to all sockets
+    emit(event, payload) {
+        this.socketServer.sockets.emit(event, payload);
+    }
+}
+// Likely to be more socket instances in future
+let ClientAppSocket;
+exports.ClientAppSocket = ClientAppSocket;
+const initialise = (server) => {
+    exports.ClientAppSocket = ClientAppSocket = new Websocket(server, "/socket/client");
+};
+exports.initialise = initialise;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.15-alpha.0",
+  "version": "1.3.15-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.3.15-alpha.0",
-    "@budibase/client": "1.3.15-alpha.0",
-    "@budibase/pro": "1.3.13",
-    "@budibase/string-templates": "1.3.15-alpha.0",
-    "@budibase/types": "1.3.15-alpha.0",
+    "@budibase/backend-core": "1.3.15-alpha.3",
+    "@budibase/client": "1.3.15-alpha.3",
+    "@budibase/pro": "1.3.15-alpha.2",
+    "@budibase/string-templates": "1.3.15-alpha.3",
+    "@budibase/types": "1.3.15-alpha.3",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -139,6 +139,7 @@
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
+    "socket.io": "^4.5.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
     "tar": "6.1.11",
@@ -198,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "38f3c6d1d713f5e1bd18fdd81bdffb9bf7d4f9d3"
+  "gitHead": "a1eafcef980f6ed36933a6d2953e35b0de82be51"
 }

package/src/api/controllers/plugin/file.ts

@@ -0,0 +1,15 @@
+import {
+  createTempFolder,
+  getPluginMetadata,
+  extractTarball,
+} from "../../../utilities/fileSystem"
+
+export async function fileUpload(file: { name: string; path: string }) {
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/github.ts

@@ -0,0 +1,75 @@
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { downloadUnzipTarball } from "./utils"
+
+export async function request(
+  url: string,
+  headers: { [key: string]: string },
+  err: string
+) {
+  const response = await fetch(url, { headers })
+  if (response.status >= 300) {
+    const respErr = await response.text()
+    throw new Error(`Error: ${err} - ${respErr}`)
+  }
+  return response.json()
+}
+
+export async function githubUpload(url: string, name = "", token = "") {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
+  const pluginDetails = await request(
+    githubApiUrl,
+    headers,
+    "Repository not found"
+  )
+  const pluginName = pluginDetails.name || name
+  const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+    ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+    : undefined
+  if (!pluginLatestReleaseUrl) {
+    throw new Error("Github release not found")
+  }
+
+  const pluginReleaseDetails = await request(
+    pluginLatestReleaseUrl,
+    headers,
+    "Github latest release not found"
+  )
+  const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+    (x: any) => x?.["content_type"] === "application/gzip"
+  )
+  const pluginLastReleaseTarballUrl =
+    pluginReleaseTarballAsset?.["browser_download_url"]
+  if (!pluginLastReleaseTarballUrl) {
+    throw new Error("Github latest release url not found")
+  }
+
+  try {
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+    return await getPluginMetadata(path)
+  } catch (err: any) {
+    let errMsg = err?.message || err
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarball not found"
+    }
+    throw new Error(errMsg)
+  }
+}

package/src/api/controllers/plugin/index.ts

@@ -0,0 +1,206 @@
+import { ObjectStoreBuckets } from "../../../constants"
+import { loadJSFile } from "../../../utilities/fileSystem"
+import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
+import { validate } from "@budibase/backend-core/plugins"
+import { generatePluginID, getPluginParams } from "../../../db/utils"
+import {
+  uploadDirectory,
+  deleteFolder,
+} from "@budibase/backend-core/objectStore"
+import { PluginType, FileType, PluginSource } from "@budibase/types"
+import env from "../../../environment"
+import { ClientAppSocket } from "../../../websocket"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function upload(ctx: any) {
+  const plugins: FileType[] =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
+  try {
+    let docs = []
+    // can do single or multiple plugins
+    for (let plugin of plugins) {
+      const doc = await processPlugin(plugin, PluginSource.FILE)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function create(ctx: any) {
+  const { source, url, headers, githubToken } = ctx.request.body
+
+  if (!env.SELF_HOSTED) {
+    ctx.throw(400, "Plugins not supported outside of self-host.")
+  }
+
+  try {
+    let metadata
+    let directory
+    // Generating random name as a backup and needed for url
+    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+
+    switch (source) {
+      case PluginSource.NPM:
+        const { metadata: metadataNpm, directory: directoryNpm } =
+          await npmUpload(url, name)
+        metadata = metadataNpm
+        directory = directoryNpm
+        break
+      case PluginSource.GITHUB:
+        const { metadata: metadataGithub, directory: directoryGithub } =
+          await githubUpload(url, name, githubToken)
+        metadata = metadataGithub
+        directory = directoryGithub
+        break
+      case PluginSource.URL:
+        const headersObj = headers || {}
+        const { metadata: metadataUrl, directory: directoryUrl } =
+          await urlUpload(url, name, headersObj)
+        metadata = metadataUrl
+        directory = directoryUrl
+        break
+    }
+
+    validate(metadata?.schema)
+
+    const doc = await storePlugin(metadata, directory, source)
+
+    ctx.body = {
+      message: "Plugin uploaded successfully",
+      plugins: [doc],
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+  ctx.status = 200
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {
+  const db = getGlobalDB()
+  const { pluginId } = ctx.params
+
+  try {
+    const plugin = await db.get(pluginId)
+    const bucketPath = `${plugin.name}/`
+    await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath)
+
+    await db.remove(pluginId, plugin._rev)
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to delete plugin: ${errMsg}`)
+  }
+
+  ctx.message = `Plugin ${ctx.params.pluginId} deleted.`
+  ctx.status = 200
+}
+
+export async function storePlugin(
+  metadata: any,
+  directory: any,
+  source?: string
+) {
+  const db = getGlobalDB()
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description,
+    hash = metadata.schema.hash
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  let doc = {
+    _id: pluginId,
+    _rev: rev,
+    ...metadata,
+    name,
+    version,
+    hash,
+    description,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+
+  if (source) {
+    doc = {
+      ...doc,
+      source,
+    }
+  }
+
+  const response = await db.put(doc)
+  ClientAppSocket.emit("plugin-update", { name, hash })
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}
+
+export async function processPlugin(plugin: FileType, source?: string) {
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
+
+  const { metadata, directory } = await fileUpload(plugin)
+  return await storePlugin(metadata, directory, source)
+}

package/src/api/controllers/plugin/npm.ts

@@ -0,0 +1,56 @@
+import {
+  getPluginMetadata,
+  findFileRec,
+  extractTarball,
+  deleteFolderFileSystem,
+} from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { join } from "path"
+import { downloadUnzipTarball } from "./utils"
+
+export async function npmUpload(url: string, name: string, headers = {}) {
+  let npmTarballUrl = url
+  let pluginName = name
+
+  if (
+    !npmTarballUrl.includes("https://www.npmjs.com") &&
+    !npmTarballUrl.includes("https://registry.npmjs.org")
+  ) {
+    throw new Error("The plugin origin must be from NPM")
+  }
+
+  if (!npmTarballUrl.includes(".tgz")) {
+    const npmPackageURl = url.replace(
+      "https://www.npmjs.com/package/",
+      "https://registry.npmjs.org/"
+    )
+    const response = await fetch(npmPackageURl)
+    if (response.status !== 200) {
+      throw new Error("NPM Package not found")
+    }
+
+    let npmDetails = await response.json()
+    pluginName = npmDetails.name
+    const npmVersion = npmDetails["dist-tags"].latest
+    npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
+
+    if (!npmTarballUrl) {
+      throw new Error("NPM tarball url not found")
+    }
+  }
+
+  const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
+  const tarballPluginFile = findFileRec(path, ".tar.gz")
+  if (!tarballPluginFile) {
+    throw new Error("Tarball plugin file not found")
+  }
+
+  try {
+    await extractTarball(tarballPluginFile, path)
+    deleteFolderFileSystem(join(path, "package"))
+  } catch (err: any) {
+    throw new Error(err)
+  }
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/uploaders.ts

@@ -0,0 +1,4 @@
+export { fileUpload } from "./file"
+export { githubUpload } from "./github"
+export { npmUpload } from "./npm"
+export { urlUpload } from "./url"

package/src/api/controllers/plugin/url.ts

@@ -0,0 +1,12 @@
+import { downloadUnzipTarball } from "./utils"
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+
+export async function urlUpload(url: string, name = "", headers = {}) {
+  if (!url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  const path = await downloadUnzipTarball(url, name, headers)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/utils.ts

@@ -0,0 +1,19 @@
+import {
+  createTempFolder,
+  downloadTarballDirect,
+} from "../../../utilities/fileSystem"
+
+export async function downloadUnzipTarball(
+  url: string,
+  name: string,
+  headers = {}
+) {
+  try {
+    const path = createTempFolder(name)
+    await downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e: any) {
+    throw new Error(e.message)
+  }
+}

package/src/api/controllers/screen.js

@@ -71,6 +71,7 @@ exports.save = async ctx => {
           name: plugin.name,
           version: plugin.version,
           jsUrl: plugin.jsUrl,
+          hash: plugin.hash,
         })
       }
     })

package/src/api/controllers/static/templates/preview.hbs

@@ -67,7 +67,8 @@
         previewDevice,
         navigation,
         hiddenComponentIds,
-        usedPlugins
+        usedPlugins,
+        location
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -83,6 +84,7 @@
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
       window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
+      window["##BUDIBASE_LOCATION##"] = location
 
       // Initialise app
       try {

package/src/api/routes/plugin.ts

@@ -7,6 +7,7 @@ const router = new Router()
 
 router
   .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
+  .post("/api/plugin", authorized(BUILDER), controller.create)
   .get("/api/plugin", authorized(BUILDER), controller.fetch)
   .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
 

package/src/app.ts

@@ -32,6 +32,7 @@ import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
 import { watch } from "./watch"
+import { initialise as initialiseWebsockets } from "./websocket"
 
 const app = new Koa()
 
@@ -76,6 +77,7 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
+initialiseWebsockets(server)
 
 let shuttingDown = false,
   errCode = 0

package/src/integrations/base/sqlTable.ts

@@ -96,7 +96,9 @@ function generateSchema(
     const deletedColumns = Object.entries(oldTable.schema)
       .filter(
         ([key, schema]) =>
-          schema.type !== FieldTypes.LINK && table.schema[key] == null
+          schema.type !== FieldTypes.LINK &&
+          schema.type !== FieldTypes.FORMULA &&
+          table.schema[key] == null
       )
       .map(([key]) => key)
     deletedColumns.forEach(key => {

package/src/module.d.ts

@@ -11,3 +11,4 @@ declare module "@budibase/backend-core/encryption"
 declare module "@budibase/backend-core/utils"
 declare module "@budibase/backend-core/redis"
 declare module "@budibase/backend-core/objectStore"
+declare module "@budibase/backend-core/plugins"