@budibase/server 1.3.11 → 1.3.12-alpha.2

package/dist/api/controllers/application.js

@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sync = exports.destroy = exports.revertClient = exports.updateClient = exports.update = exports.create = exports.fetchAppPackage = exports.fetchAppDefinition = exports.fetch = exports.getAppUrl = void 0;
+exports.updateAppPackage = exports.sync = exports.destroy = exports.revertClient = exports.updateClient = exports.update = exports.create = exports.fetchAppPackage = exports.fetchAppDefinition = exports.fetch = exports.getAppUrl = void 0;
 const environment_1 = __importDefault(require("../../environment"));
 const package_json_1 = __importDefault(require("../../../package.json"));
 const staticViews_1 = require("../../db/views/staticViews");
@@ -353,7 +353,7 @@ const update = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         checkAppUrl(ctx, apps, url, ctx.params.appId);
         ctx.request.body.url = url;
     }
-    const app = yield updateAppPackage(ctx.request.body, ctx.params.appId);
+    const app = yield (0, exports.updateAppPackage)(ctx.request.body, ctx.params.appId);
     yield backend_core_2.events.app.updated(app);
     ctx.status = 200;
     ctx.body = app;
@@ -375,7 +375,7 @@ const updateClient = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         version: updatedToVersion,
         revertableVersion: currentVersion,
     };
-    const app = yield updateAppPackage(appPackageUpdates, ctx.params.appId);
+    const app = yield (0, exports.updateAppPackage)(appPackageUpdates, ctx.params.appId);
     yield backend_core_2.events.app.versionUpdated(app, currentVersion, updatedToVersion);
     ctx.status = 200;
     ctx.body = app;
@@ -399,7 +399,7 @@ const revertClient = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         version: revertedToVersion,
         revertableVersion: null,
     };
-    const app = yield updateAppPackage(appPackageUpdates, ctx.params.appId);
+    const app = yield (0, exports.updateAppPackage)(appPackageUpdates, ctx.params.appId);
     yield backend_core_2.events.app.versionReverted(app, currentVersion, revertedToVersion);
     ctx.status = 200;
     ctx.body = app;
@@ -531,6 +531,7 @@ const updateAppPackage = (appPackage, appId) => __awaiter(void 0, void 0, void 0
         return newAppPackage;
     }));
 });
+exports.updateAppPackage = updateAppPackage;
 const migrateAppNavigation = () => __awaiter(void 0, void 0, void 0, function* () {
     const db = backend_core_1.context.getAppDB();
     const existing = yield db.get(utils_1.DocumentType.APP_METADATA);

package/dist/api/controllers/component.js

@@ -8,9 +8,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { DocumentType } = require("../../db/utils");
+const { DocumentType, getPluginParams } = require("../../db/utils");
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem");
 const { getAppDB } = require("@budibase/backend-core/context");
+const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+const env = require("../../environment");
 exports.fetchAppComponentDefinitions = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
@@ -35,6 +37,21 @@ exports.fetchAppComponentDefinitions = function (ctx) {
                     }
                 }
             }
+            // for now custom components only supported in self-host
+            if (env.SELF_HOSTED) {
+                // Add custom components
+                const globalDB = getGlobalDB();
+                const response = yield globalDB.allDocs(getPluginParams(null, {
+                    include_docs: true,
+                }));
+                response.rows
+                    .map(row => row.doc)
+                    .filter(plugin => plugin.schema.type === "component")
+                    .forEach(plugin => {
+                    const fullComponentName = `plugin/${plugin.name}`;
+                    definitions[fullComponentName] = Object.assign({ component: fullComponentName }, plugin.schema.schema);
+                });
+            }
             ctx.body = definitions;
         }
         catch (err) {

package/dist/api/controllers/datasource.js

@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const { generateDatasourceID, getDatasourceParams, getQueryParams, DocumentType, BudibaseInternalDB, getTableParams, } = require("../../db/utils");
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants");
-const { integrations } = require("../../integrations");
+const { getIntegration } = require("../../integrations");
 const { getDatasourceAndQuery } = require("./row/utils");
 const { invalidateDynamicVariables } = require("../../threads/utils");
 const { getAppDB } = require("@budibase/backend-core/context");
@@ -96,7 +96,7 @@ exports.update = function (ctx) {
         datasource._rev = response.rev;
         // Drain connection pools when configuration is changed
         if (datasource.source) {
-            const source = integrations[datasource.source];
+            const source = yield getIntegration(datasource.source);
             if (source && source.pool) {
                 yield source.pool.end();
             }
@@ -123,7 +123,7 @@ exports.save = function (ctx) {
         datasource._rev = dbResp.rev;
         // Drain connection pools when configuration is changed
         if (datasource.source) {
-            const source = integrations[datasource.source];
+            const source = yield getIntegration(datasource.source);
             if (source && source.pool) {
                 yield source.pool.end();
             }
@@ -188,7 +188,7 @@ function updateError(error, newError, tables) {
     return error;
 }
 const buildSchemaHelper = (datasource) => __awaiter(void 0, void 0, void 0, function* () {
-    const Connector = integrations[datasource.source];
+    const Connector = yield getIntegration(datasource.source);
     // Connect to the DB and build the schema
     const connector = new Connector(datasource.config);
     yield connector.buildSchema(datasource._id, datasource.entities);

package/dist/api/controllers/integration.js

@@ -8,15 +8,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { cloneDeep } = require("lodash");
-const { definitions } = require("../../integrations");
+const { getDefinitions } = require("../../integrations");
 const { SourceName } = require("@budibase/types");
 const googlesheets = require("../../integrations/googlesheets");
 const { featureFlags } = require("@budibase/backend-core");
 exports.fetch = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         ctx.status = 200;
-        const defs = cloneDeep(definitions);
+        const defs = yield getDefinitions();
         // for google sheets integration google verification
         if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
             defs[SourceName.GOOGLE_SHEETS] = googlesheets.schema;
@@ -26,7 +25,8 @@ exports.fetch = function (ctx) {
 };
 exports.find = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
+        const defs = yield getDefinitions();
         ctx.status = 200;
-        ctx.body = definitions[ctx.params.type];
+        ctx.body = defs[ctx.params.type];
     });
 };

package/dist/api/controllers/plugin.js

@@ -0,0 +1,119 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processPlugin = exports.destroy = exports.fetch = exports.upload = exports.getPlugins = void 0;
+const constants_1 = require("../../constants");
+const fileSystem_1 = require("../../utilities/fileSystem");
+const tenancy_1 = require("@budibase/backend-core/tenancy");
+const utils_1 = require("../../db/utils");
+const objectStore_1 = require("@budibase/backend-core/objectStore");
+const types_1 = require("@budibase/types");
+const environment_1 = __importDefault(require("../../environment"));
+function getPlugins(type) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const db = (0, tenancy_1.getGlobalDB)();
+        const response = yield db.allDocs((0, utils_1.getPluginParams)(null, {
+            include_docs: true,
+        }));
+        const plugins = response.rows.map((row) => row.doc);
+        if (type) {
+            return plugins.filter((plugin) => { var _a; return ((_a = plugin.schema) === null || _a === void 0 ? void 0 : _a.type) === type; });
+        }
+        else {
+            return plugins;
+        }
+    });
+}
+exports.getPlugins = getPlugins;
+function upload(ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const plugins = ctx.request.files.file.length > 1
+            ? Array.from(ctx.request.files.file)
+            : [ctx.request.files.file];
+        try {
+            let docs = [];
+            // can do single or multiple plugins
+            for (let plugin of plugins) {
+                const doc = yield processPlugin(plugin);
+                docs.push(doc);
+            }
+            ctx.body = {
+                message: "Plugin(s) uploaded successfully",
+                plugins: docs,
+            };
+        }
+        catch (err) {
+            const errMsg = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
+            ctx.throw(400, `Failed to import plugin: ${errMsg}`);
+        }
+    });
+}
+exports.upload = upload;
+function fetch(ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        ctx.body = yield getPlugins();
+    });
+}
+exports.fetch = fetch;
+function destroy(ctx) {
+    return __awaiter(this, void 0, void 0, function* () { });
+}
+exports.destroy = destroy;
+function processPlugin(plugin) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!environment_1.default.SELF_HOSTED) {
+            throw new Error("Plugins not supported outside of self-host.");
+        }
+        const db = (0, tenancy_1.getGlobalDB)();
+        const { metadata, directory } = yield (0, fileSystem_1.extractPluginTarball)(plugin);
+        const version = metadata.package.version, name = metadata.package.name, description = metadata.package.description;
+        // first open the tarball into tmp directory
+        const bucketPath = `${name}/`;
+        const files = yield (0, objectStore_1.uploadDirectory)(constants_1.ObjectStoreBuckets.PLUGINS, directory, bucketPath);
+        const jsFile = files.find((file) => file.name.endsWith(".js"));
+        if (!jsFile) {
+            throw new Error(`Plugin missing .js file.`);
+        }
+        // validate the JS for a datasource
+        if (metadata.schema.type === types_1.PluginType.DATASOURCE) {
+            const js = (0, fileSystem_1.loadJSFile)(directory, jsFile.name);
+            // TODO: this isn't safe - but we need full node environment
+            // in future we should do this in a thread for safety
+            try {
+                eval(js);
+            }
+            catch (err) {
+                const message = (err === null || err === void 0 ? void 0 : err.message) ? err.message : JSON.stringify(err);
+                throw new Error(`JS invalid: ${message}`);
+            }
+        }
+        const jsFileName = jsFile.name;
+        const pluginId = (0, utils_1.generatePluginID)(name);
+        // overwrite existing docs entirely if they exist
+        let rev;
+        try {
+            const existing = yield db.get(pluginId);
+            rev = existing._rev;
+        }
+        catch (err) {
+            rev = undefined;
+        }
+        const doc = Object.assign(Object.assign({ _id: pluginId, _rev: rev }, metadata), { name,
+            version,
+            description, jsUrl: `${bucketPath}${jsFileName}` });
+        const response = yield db.put(doc);
+        return Object.assign(Object.assign({}, doc), { _rev: response.rev });
+    });
+}
+exports.processPlugin = processPlugin;

package/dist/api/controllers/screen.js

@@ -8,10 +8,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { getScreenParams, generateScreenID } = require("../../db/utils");
+const { getScreenParams, generateScreenID, getPluginParams, DocumentType, } = require("../../db/utils");
 const { AccessController } = require("@budibase/backend-core/roles");
 const { getAppDB } = require("@budibase/backend-core/context");
 const { events } = require("@budibase/backend-core");
+const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+const { updateAppPackage } = require("./application");
 exports.fetch = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     const db = getAppDB();
     const screens = (yield db.allDocs(getScreenParams(null, {
@@ -28,11 +30,44 @@ exports.save = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         eventFn = events.screen.created;
     }
     const response = yield db.put(screen);
+    // Find any custom components being used
+    let pluginNames = [];
+    let pluginAdded = false;
+    findPlugins(screen.props, pluginNames);
+    if (pluginNames.length) {
+        const globalDB = getGlobalDB();
+        const pluginsResponse = yield globalDB.allDocs(getPluginParams(null, {
+            include_docs: true,
+        }));
+        const requiredPlugins = pluginsResponse.rows
+            .map(row => row.doc)
+            .filter(plugin => {
+            return (plugin.schema.type === "component" &&
+                pluginNames.includes(`plugin/${plugin.name}`));
+        });
+        // Update the app metadata
+        const application = yield db.get(DocumentType.APP_METADATA);
+        let usedPlugins = application.usedPlugins || [];
+        requiredPlugins.forEach(plugin => {
+            if (!usedPlugins.find(x => x._id === plugin._id)) {
+                pluginAdded = true;
+                usedPlugins.push({
+                    _id: plugin._id,
+                    name: plugin.name,
+                    version: plugin.version,
+                    jsUrl: plugin.jsUrl,
+                });
+            }
+        });
+        if (pluginAdded) {
+            yield updateAppPackage({ usedPlugins }, ctx.appId);
+        }
+    }
     if (eventFn) {
         yield eventFn(screen);
     }
     ctx.message = `Screen ${screen.name} saved.`;
-    ctx.body = Object.assign(Object.assign({}, screen), { _id: response.id, _rev: response.rev });
+    ctx.body = Object.assign(Object.assign({}, screen), { _id: response.id, _rev: response.rev, pluginAdded });
 });
 exports.destroy = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     const db = getAppDB();
@@ -45,3 +80,17 @@ exports.destroy = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     };
     ctx.status = 200;
 });
+const findPlugins = (component, foundPlugins) => {
+    if (!component) {
+        return;
+    }
+    if (component._component.startsWith("plugin")) {
+        if (!foundPlugins.includes(component._component)) {
+            foundPlugins.push(component._component);
+        }
+    }
+    if (!component._children || !component._children.length) {
+        return;
+    }
+    component._children.forEach(child => findPlugins(child, foundPlugins));
+};

package/dist/api/controllers/static/index.js

@@ -113,6 +113,7 @@ const serveApp = function (ctx) {
                 production: env.isProd(),
                 appId,
                 clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+                usedPlugins: appInfo.usedPlugins,
             });
             const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`);
             ctx.body = yield processString(appHbs, {

package/dist/api/controllers/static/templates/BudibaseApp.svelte

@@ -3,6 +3,7 @@
   export let favicon = ""
 
   export let clientLibPath
+  export let usedPlugins
 </script>
 
 <svelte:head>
@@ -83,6 +84,15 @@
   </script>
   <script type="application/javascript" src={clientLibPath}>
   </script>
+  <!-- Custom components need inserted after the core client library -->
+  <!-- But before loadBudibase is called -->
+  {#if usedPlugins?.length}
+    {#each usedPlugins as plugin}
+      <script
+        type="application/javascript"
+        src={`/plugins/${plugin.jsUrl}`}></script>
+    {/each}
+  {/if}
   <script type="application/javascript">
     if (window.loadBudibase) {
       window.loadBudibase()

package/dist/api/controllers/static/templates/preview.hbs

@@ -66,7 +66,8 @@
         customTheme,
         previewDevice,
         navigation,
-        hiddenComponentIds
+        hiddenComponentIds,
+        usedPlugins
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -81,6 +82,7 @@
       window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
+      window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
 
       // Initialise app
       try {

package/dist/api/routes/index.js

@@ -30,6 +30,7 @@ const metadata_1 = __importDefault(require("./metadata"));
 const dev_1 = __importDefault(require("./dev"));
 const cloud_1 = __importDefault(require("./cloud"));
 const migrations_1 = __importDefault(require("./migrations"));
+const plugin_1 = __importDefault(require("./plugin"));
 var static_1 = require("./static");
 Object.defineProperty(exports, "staticRoutes", { enumerable: true, get: function () { return __importDefault(static_1).default; } });
 var public_1 = require("./public");
@@ -63,4 +64,5 @@ exports.mainRoutes = [
     table_1.default,
     row_1.default,
     migrations_1.default,
+    plugin_1.default,
 ];

package/dist/api/routes/plugin.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const router_1 = __importDefault(require("@koa/router"));
+const controller = __importStar(require("../controllers/plugin"));
+const authorized_1 = __importDefault(require("../../middleware/authorized"));
+const permissions_1 = require("@budibase/backend-core/permissions");
+const router = new router_1.default();
+router
+    .post("/api/plugin/upload", (0, authorized_1.default)(permissions_1.BUILDER), controller.upload)
+    .get("/api/plugin", (0, authorized_1.default)(permissions_1.BUILDER), controller.fetch)
+    .delete("/api/plugin/:pluginId", (0, authorized_1.default)(permissions_1.BUILDER), controller.destroy);
+exports.default = router;

package/dist/app.js

@@ -60,10 +60,12 @@ const bullboard = require("./automations/bullboard");
 const { logAlert } = require("@budibase/backend-core/logging");
 const { pinoSettings } = require("@budibase/backend-core");
 const { Thread } = require("./threads");
+const fs = require("fs");
 const redis_1 = __importDefault(require("./utilities/redis"));
 const migrations = __importStar(require("./migrations"));
 const backend_core_1 = require("@budibase/backend-core");
 const workerRequests_1 = require("./utilities/workerRequests");
+const watch_1 = require("./watch");
 const app = new Koa();
 // set up top level koa middleware
 app.use(koaBody({
@@ -150,6 +152,13 @@ module.exports = server.listen(env.PORT || 0, () => __awaiter(void 0, void 0, vo
             }
         }
     }
+    // monitor plugin directory if required
+    if (env.SELF_HOSTED &&
+        !env.MULTI_TENANCY &&
+        env.PLUGINS_DIR &&
+        fs.existsSync(env.PLUGINS_DIR)) {
+        (0, watch_1.watch)();
+    }
     // check for version updates
     yield backend_core_1.installation.checkInstallVersion();
     // done last - this will never complete

package/dist/db/utils.js

@@ -7,7 +7,7 @@ const AppStatus = {
     ALL: "all",
     DEPLOYED: "published",
 };
-const DocumentType = Object.assign(Object.assign({}, CoreDocTypes), { TABLE: "ta", ROW: "ro", USER: "us", AUTOMATION: "au", LINK: "li", WEBHOOK: "wh", INSTANCE: "inst", LAYOUT: "layout", SCREEN: "screen", DATASOURCE: "datasource", DATASOURCE_PLUS: "datasource_plus", QUERY: "query", DEPLOYMENTS: "deployments", METADATA: "metadata", MEM_VIEW: "view", USER_FLAG: "flag", AUTOMATION_METADATA: "meta_au" });
+const DocumentType = Object.assign(Object.assign({}, CoreDocTypes), { TABLE: "ta", ROW: "ro", USER: "us", AUTOMATION: "au", LINK: "li", WEBHOOK: "wh", INSTANCE: "inst", LAYOUT: "layout", SCREEN: "screen", DATASOURCE: "datasource", DATASOURCE_PLUS: "datasource_plus", QUERY: "query", DEPLOYMENTS: "deployments", METADATA: "metadata", MEM_VIEW: "view", USER_FLAG: "flag", AUTOMATION_METADATA: "meta_au", PLUGIN: "plg" });
 const InternalTables = {
     USER_METADATA: "ta_users",
 };
@@ -274,6 +274,9 @@ exports.generateMemoryViewID = viewName => {
 exports.getMemoryViewParams = (otherProps = {}) => {
     return getDocParams(DocumentType.MEM_VIEW, null, otherProps);
 };
+exports.generatePluginID = name => {
+    return `${DocumentType.PLUGIN}${SEPARATOR}${name}`;
+};
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */
@@ -283,3 +286,9 @@ exports.getMultiIDParams = ids => {
         include_docs: true,
     };
 };
+/**
+ * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
+ */
+exports.getPluginParams = (pluginId = null, otherProps = {}) => {
+    return getDocParams(DocumentType.PLUGIN, pluginId, otherProps);
+};

package/dist/environment.js

@@ -64,6 +64,7 @@ module.exports = {
     SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
     BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
     BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+    PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
     // flags
     ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
     DISABLE_THREADING: process.env.DISABLE_THREADING,