@budibase/server 1.3.17 → 1.3.19-alpha.0

package/dist/utilities/fileSystem/index.js

@@ -14,14 +14,18 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
+const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
+const tar = require("tar");
+const fetch = require("node-fetch");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -96,6 +100,12 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
 exports.loadHandlebarsFile = path => {
     return fs.readFileSync(path, "utf8");
 };
+/**
+ * Same as above just with a different name.
+ */
+exports.loadJSFile = (directory, name) => {
+    return fs.readFileSync(join(directory, name), "utf8");
+};
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -294,6 +304,104 @@ exports.cleanup = appIds => {
         }
     }
 };
+const createTempFolder = item => {
+    const path = join(budibaseTempDir(), item);
+    try {
+        // remove old tmp directories automatically - don't combine
+        if (fs.existsSync(path)) {
+            fs.rmSync(path, { recursive: true, force: true });
+        }
+        fs.mkdirSync(path);
+    }
+    catch (err) {
+        throw new Error(`Path cannot be created: ${err.message}`);
+    }
+    return path;
+};
+exports.createTempFolder = createTempFolder;
+const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
+    yield tar.extract({
+        file: fromFilePath,
+        C: toPath,
+    });
+});
+exports.extractTarball = extractTarball;
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
+    let metadata = {};
+    try {
+        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
+        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
+        metadata.schema = JSON.parse(schema);
+        metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
+        }
+    }
+    catch (err) {
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
+    }
+    return { metadata, directory: path };
+});
+exports.getPluginMetadata = getPluginMetadata;
+exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!fs.existsSync(DATASOURCE_PATH)) {
+        fs.mkdirSync(DATASOURCE_PATH);
+    }
+    const filename = join(DATASOURCE_PATH, name);
+    const metadataName = `${filename}.bbmetadata`;
+    if (fs.existsSync(filename)) {
+        const currentHash = fs.readFileSync(metadataName, "utf8");
+        // if hash is the same return the file, otherwise remove it and re-download
+        if (currentHash === hash) {
+            return require(filename);
+        }
+        else {
+            console.log(`Updating plugin: ${name}`);
+            fs.unlinkSync(filename);
+        }
+    }
+    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
+    const response = yield fetch(fullUrl);
+    if (response.status === 200) {
+        const content = yield response.text();
+        fs.writeFileSync(filename, content);
+        fs.writeFileSync(metadataName, hash);
+        return require(filename);
+    }
+    else {
+        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
+    }
+});
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+    if (!fs.existsSync(startPath)) {
+        return;
+    }
+    const files = fs.readdirSync(startPath);
+    for (let i = 0, len = files.length; i < len; i++) {
+        const filename = join(startPath, files[i]);
+        const stat = fs.lstatSync(filename);
+        if (stat.isDirectory()) {
+            return exports.findFileRec(filename, filter);
+        }
+        else if (filename.endsWith(filter)) {
+            return filename;
+        }
+    }
+};
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+    if (!fs.existsSync(path)) {
+        return;
+    }
+    fs.rmSync(path, { recursive: true, force: true });
+};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -301,5 +409,6 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
+exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/dist/watch.js

@@ -0,0 +1,77 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.watch = void 0;
+const path_1 = __importDefault(require("path"));
+const env = __importStar(require("./environment"));
+const chokidar_1 = __importDefault(require("chokidar"));
+const fs_1 = __importDefault(require("fs"));
+const backend_core_1 = require("@budibase/backend-core");
+const constants_1 = require("@budibase/backend-core/constants");
+const plugin_1 = require("./api/controllers/plugin");
+function watch() {
+    const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
+    chokidar_1.default
+        .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: {
+            pollInterval: 100,
+            stabilityThreshold: 250,
+        },
+        usePolling: true,
+        interval: 250,
+    })
+        .on("all", (event, path) => __awaiter(this, void 0, void 0, function* () {
+        // Sanity checks
+        if (!(path === null || path === void 0 ? void 0 : path.endsWith(".tar.gz")) || !fs_1.default.existsSync(path)) {
+            return;
+        }
+        yield backend_core_1.tenancy.doInTenant(constants_1.DEFAULT_TENANT_ID, () => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const split = path.split("/");
+                const name = split[split.length - 1];
+                console.log("Importing plugin:", path);
+                yield (0, plugin_1.processPlugin)({ name, path });
+            }
+            catch (err) {
+                const message = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
+                console.error("Failed to import plugin:", message);
+            }
+        }));
+    }));
+}
+exports.watch = watch;

package/dist/websocket.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClientAppSocket = exports.initialise = void 0;
+const socket_io_1 = require("socket.io");
+class Websocket {
+    constructor(server, path) {
+        this.socketServer = new socket_io_1.Server(server, {
+            path,
+        });
+    }
+    // Emit an event to all sockets
+    emit(event, payload) {
+        this.socketServer.sockets.emit(event, payload);
+    }
+}
+// Likely to be more socket instances in future
+let ClientAppSocket;
+exports.ClientAppSocket = ClientAppSocket;
+const initialise = (server) => {
+    exports.ClientAppSocket = ClientAppSocket = new Websocket(server, "/socket/client");
+};
+exports.initialise = initialise;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.17",
+  "version": "1.3.19-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.3.17",
-    "@budibase/client": "^1.3.17",
-    "@budibase/pro": "1.3.16",
-    "@budibase/string-templates": "^1.3.17",
-    "@budibase/types": "^1.3.17",
+    "@budibase/backend-core": "1.3.19-alpha.0",
+    "@budibase/client": "1.3.19-alpha.0",
+    "@budibase/pro": "1.3.18",
+    "@budibase/string-templates": "1.3.19-alpha.0",
+    "@budibase/types": "1.3.19-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
+    "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
@@ -138,8 +139,10 @@
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
+    "socket.io": "^4.5.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
+    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -196,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "01d7c9d994aa66a1d08ab71acc9e6c8eab825a47"
+  "gitHead": "04749bf93a17516f8f71d44b49995b2d50b080bb"
 }

package/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/specs/openapi.json

@@ -1327,7 +1327,7 @@
             "type": "string"
           },
           "datasourceId": {
-            "description": "The ID of the data source the query belongs to.",
+            "description": "The ID of the datasource the query belongs to.",
             "type": "string"
           },
           "parameters": {
@@ -1386,7 +1386,7 @@
                   "type": "string"
                 },
                 "datasourceId": {
-                  "description": "The ID of the data source the query belongs to.",
+                  "description": "The ID of the datasource the query belongs to.",
                   "type": "string"
                 },
                 "parameters": {
@@ -2289,7 +2289,7 @@
         },
         "responses": {
           "200": {
-            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external data sources.",
+            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external datasources.",
             "content": {
               "application/json": {
                 "schema": {

package/specs/openapi.yaml

@@ -1004,7 +1004,7 @@ components:
           description: The ID of the query.
           type: string
         datasourceId:
-          description: The ID of the data source the query belongs to.
+          description: The ID of the datasource the query belongs to.
           type: string
         parameters:
           description: The bindings which are required to perform this query.
@@ -1051,7 +1051,7 @@ components:
                 description: The ID of the query.
                 type: string
               datasourceId:
-                description: The ID of the data source the query belongs to.
+                description: The ID of the datasource the query belongs to.
                 type: string
               parameters:
                 description: The bindings which are required to perform this query.
@@ -1665,7 +1665,7 @@ paths:
       responses:
         "200":
           description: Returns the created table, including the ID which has been
-            generated for it. This can be internal or external data sources.
+            generated for it. This can be internal or external datasources.
           content:
             application/json:
               schema:

package/specs/resources/query.js

@@ -82,7 +82,7 @@ const querySchema = object(
       type: "string",
     },
     datasourceId: {
-      description: "The ID of the data source the query belongs to.",
+      description: "The ID of the datasource the query belongs to.",
       type: "string",
     },
     parameters: {

package/src/api/controllers/application.ts

@@ -572,7 +572,7 @@ export const sync = async (ctx: any, next: any) => {
   }
 }
 
-const updateAppPackage = async (appPackage: any, appId: any) => {
+export const updateAppPackage = async (appPackage: any, appId: any) => {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/component.js

@@ -1,6 +1,7 @@
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, getPluginParams } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   try {
@@ -10,7 +11,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
     let componentManifests = await Promise.all(
       app.componentLibraries.map(async library => {
         let manifest = await getComponentLibraryManifest(library)
-
         return {
           manifest,
           library,
@@ -31,6 +31,25 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
+
+    // Add custom components
+    const globalDB = getGlobalDB()
+    const response = await globalDB.allDocs(
+      getPluginParams(null, {
+        include_docs: true,
+      })
+    )
+    response.rows
+      .map(row => row.doc)
+      .filter(plugin => plugin.schema.type === "component")
+      .forEach(plugin => {
+        const fullComponentName = `plugin/${plugin.name}`
+        definitions[fullComponentName] = {
+          component: fullComponentName,
+          ...plugin.schema.schema,
+        }
+      })
+
     ctx.body = definitions
   } catch (err) {
     console.error(`component-definitions=failed`, err)

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -50,9 +50,23 @@ exports.fetch = async function (ctx) {
 exports.buildSchemaFromDb = async function (ctx) {
   const db = getAppDB()
   const datasource = await db.get(ctx.params.datasourceId)
+  const tablesFilter = ctx.request.body.tablesFilter
 
-  const { tables, error } = await buildSchemaHelper(datasource)
-  datasource.entities = tables
+  let { tables, error } = await buildSchemaHelper(datasource)
+  if (tablesFilter) {
+    if (!datasource.entities) {
+      datasource.entities = {}
+    }
+    for (let key in tables) {
+      if (
+        tablesFilter.some(filter => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        datasource.entities[key] = tables[key]
+      }
+    }
+  } else {
+    datasource.entities = tables
+  }
 
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
@@ -114,7 +128,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +163,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,15 +232,14 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = integrations[datasource.source]
+  const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)
-  datasource.entities = connector.tables
 
   // make sure they all have a display name selected
-  for (let entity of Object.values(datasource.entities)) {
+  for (let entity of Object.values(datasource.entities ?? {})) {
     if (entity.primaryDisplay) {
       continue
     }

package/src/api/controllers/integration.js

@@ -1,12 +1,11 @@
-const { cloneDeep } = require("lodash")
-const { definitions } = require("../../integrations")
+const { getDefinitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = cloneDeep(definitions)
+  const defs = await getDefinitions()
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
+  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = definitions[ctx.params.type]
+  ctx.body = defs[ctx.params.type]
 }

package/src/api/controllers/plugin/file.ts

@@ -0,0 +1,15 @@
+import {
+  createTempFolder,
+  getPluginMetadata,
+  extractTarball,
+} from "../../../utilities/fileSystem"
+
+export async function fileUpload(file: { name: string; path: string }) {
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/github.ts

@@ -0,0 +1,75 @@
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { downloadUnzipTarball } from "./utils"
+
+export async function request(
+  url: string,
+  headers: { [key: string]: string },
+  err: string
+) {
+  const response = await fetch(url, { headers })
+  if (response.status >= 300) {
+    const respErr = await response.text()
+    throw new Error(`Error: ${err} - ${respErr}`)
+  }
+  return response.json()
+}
+
+export async function githubUpload(url: string, name = "", token = "") {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
+  const pluginDetails = await request(
+    githubApiUrl,
+    headers,
+    "Repository not found"
+  )
+  const pluginName = pluginDetails.name || name
+  const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+    ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+    : undefined
+  if (!pluginLatestReleaseUrl) {
+    throw new Error("Github release not found")
+  }
+
+  const pluginReleaseDetails = await request(
+    pluginLatestReleaseUrl,
+    headers,
+    "Github latest release not found"
+  )
+  const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+    (x: any) => x?.["content_type"] === "application/gzip"
+  )
+  const pluginLastReleaseTarballUrl =
+    pluginReleaseTarballAsset?.["browser_download_url"]
+  if (!pluginLastReleaseTarballUrl) {
+    throw new Error("Github latest release url not found")
+  }
+
+  try {
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+    return await getPluginMetadata(path)
+  } catch (err: any) {
+    let errMsg = err?.message || err
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarball not found"
+    }
+    throw new Error(errMsg)
+  }
+}