@budibase/server 1.3.15-alpha.3 → 1.3.17

package/dist/utilities/fileSystem/index.js

@@ -14,18 +14,14 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
-const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
-const tar = require("tar");
-const fetch = require("node-fetch");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
-const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -39,7 +35,16 @@ const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 exports.init = () => {
     const tempDir = budibaseTempDir();
     if (!fs.existsSync(tempDir)) {
-        fs.mkdirSync(tempDir);
+        // some test cases fire this quickly enough that
+        // synchronous cases can end up here at the same time
+        try {
+            fs.mkdirSync(tempDir);
+        }
+        catch (err) {
+            if (!err || err.code !== "EEXIST") {
+                throw err;
+            }
+        }
     }
     const clientLibPath = join(budibaseTempDir(), "budibase-client.js");
     if (env.isTest() && !fs.existsSync(clientLibPath)) {
@@ -91,12 +96,6 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
 exports.loadHandlebarsFile = path => {
     return fs.readFileSync(path, "utf8");
 };
-/**
- * Same as above just with a different name.
- */
-exports.loadJSFile = (directory, name) => {
-    return fs.readFileSync(join(directory, name), "utf8");
-};
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -295,104 +294,6 @@ exports.cleanup = appIds => {
         }
     }
 };
-const createTempFolder = item => {
-    const path = join(budibaseTempDir(), item);
-    try {
-        // remove old tmp directories automatically - don't combine
-        if (fs.existsSync(path)) {
-            fs.rmSync(path, { recursive: true, force: true });
-        }
-        fs.mkdirSync(path);
-    }
-    catch (err) {
-        throw new Error(`Path cannot be created: ${err.message}`);
-    }
-    return path;
-};
-exports.createTempFolder = createTempFolder;
-const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
-    yield tar.extract({
-        file: fromFilePath,
-        C: toPath,
-    });
-});
-exports.extractTarball = extractTarball;
-const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
-    let metadata = {};
-    try {
-        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
-        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
-        metadata.schema = JSON.parse(schema);
-        metadata.package = JSON.parse(pkg);
-        if (!metadata.package.name ||
-            !metadata.package.version ||
-            !metadata.package.description) {
-            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
-        }
-    }
-    catch (err) {
-        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
-    }
-    return { metadata, directory: path };
-});
-exports.getPluginMetadata = getPluginMetadata;
-exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!fs.existsSync(DATASOURCE_PATH)) {
-        fs.mkdirSync(DATASOURCE_PATH);
-    }
-    const filename = join(DATASOURCE_PATH, name);
-    const metadataName = `${filename}.bbmetadata`;
-    if (fs.existsSync(filename)) {
-        const currentHash = fs.readFileSync(metadataName, "utf8");
-        // if hash is the same return the file, otherwise remove it and re-download
-        if (currentHash === hash) {
-            return require(filename);
-        }
-        else {
-            console.log(`Updating plugin: ${name}`);
-            fs.unlinkSync(filename);
-        }
-    }
-    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
-    const response = yield fetch(fullUrl);
-    if (response.status === 200) {
-        const content = yield response.text();
-        fs.writeFileSync(filename, content);
-        fs.writeFileSync(metadataName, hash);
-        require(filename);
-    }
-    else {
-        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
-    }
-});
-/**
- * Find for a file recursively from start path applying filter, return first match
- */
-exports.findFileRec = (startPath, filter) => {
-    if (!fs.existsSync(startPath)) {
-        return;
-    }
-    const files = fs.readdirSync(startPath);
-    for (let i = 0, len = files.length; i < len; i++) {
-        const filename = join(startPath, files[i]);
-        const stat = fs.lstatSync(filename);
-        if (stat.isDirectory()) {
-            return exports.findFileRec(filename, filter);
-        }
-        else if (filename.endsWith(filter)) {
-            return filename;
-        }
-    }
-};
-/**
- * Remove a folder which is not empty from the file system
- */
-exports.deleteFolderFileSystem = path => {
-    if (!fs.existsSync(path)) {
-        return;
-    }
-    fs.rmSync(path, { recursive: true, force: true });
-};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -400,6 +301,5 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
-exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.15-alpha.3",
+  "version": "1.3.17",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.3.15-alpha.3",
-    "@budibase/client": "1.3.15-alpha.3",
-    "@budibase/pro": "1.3.15-alpha.2",
-    "@budibase/string-templates": "1.3.15-alpha.3",
-    "@budibase/types": "1.3.15-alpha.3",
+    "@budibase/backend-core": "^1.3.17",
+    "@budibase/client": "^1.3.17",
+    "@budibase/pro": "1.3.16",
+    "@budibase/string-templates": "^1.3.17",
+    "@budibase/types": "^1.3.17",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -95,7 +95,6 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
-    "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
@@ -139,10 +138,8 @@
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
-    "socket.io": "^4.5.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
-    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -199,5 +196,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "a1eafcef980f6ed36933a6d2953e35b0de82be51"
+  "gitHead": "01d7c9d994aa66a1d08ab71acc9e6c8eab825a47"
 }

package/scripts/dev/manage.js

@@ -58,7 +58,6 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
-      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/src/api/controllers/application.ts

@@ -572,7 +572,7 @@ export const sync = async (ctx: any, next: any) => {
   }
 }
 
-export const updateAppPackage = async (appPackage: any, appId: any) => {
+const updateAppPackage = async (appPackage: any, appId: any) => {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/component.js

@@ -1,8 +1,6 @@
-const { DocumentType, getPluginParams } = require("../../db/utils")
+const { DocumentType } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const env = require("../../environment")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   try {
@@ -12,6 +10,7 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
     let componentManifests = await Promise.all(
       app.componentLibraries.map(async library => {
         let manifest = await getComponentLibraryManifest(library)
+
         return {
           manifest,
           library,
@@ -32,28 +31,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
-
-    // for now custom components only supported in self-host
-    if (env.SELF_HOSTED) {
-      // Add custom components
-      const globalDB = getGlobalDB()
-      const response = await globalDB.allDocs(
-        getPluginParams(null, {
-          include_docs: true,
-        })
-      )
-      response.rows
-        .map(row => row.doc)
-        .filter(plugin => plugin.schema.type === "component")
-        .forEach(plugin => {
-          const fullComponentName = `plugin/${plugin.name}`
-          definitions[fullComponentName] = {
-            component: fullComponentName,
-            ...plugin.schema.schema,
-          }
-        })
-    }
-
     ctx.body = definitions
   } catch (err) {
     console.error(`component-definitions=failed`, err)

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { getIntegration } = require("../../integrations")
+const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = await getIntegration(datasource.source)
+    const source = integrations[datasource.source]
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = await getIntegration(datasource.source)
+    const source = integrations[datasource.source]
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = await getIntegration(datasource.source)
+  const Connector = integrations[datasource.source]
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)

package/src/api/controllers/integration.js

@@ -1,11 +1,12 @@
-const { getDefinitions } = require("../../integrations")
+const { cloneDeep } = require("lodash")
+const { definitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = await getDefinitions()
+  const defs = cloneDeep(definitions)
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -16,7 +17,6 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = defs[ctx.params.type]
+  ctx.body = definitions[ctx.params.type]
 }

package/src/api/controllers/row/ExternalRequest.ts

@@ -371,7 +371,10 @@ module External {
         const toColumn = `${linkedTable.name}.${relationship.to}`
         // this is important when working with multiple relationships
         // between the same tables, don't want to overlap/multiply the relations
-        if (!relationship.through && row[fromColumn] !== row[toColumn]) {
+        if (
+          !relationship.through &&
+          row[fromColumn]?.toString() !== row[toColumn]?.toString()
+        ) {
           continue
         }
         let linked = basicProcessing(row, linkedTable)

package/src/api/controllers/screen.js

@@ -1,14 +1,7 @@
-const {
-  getScreenParams,
-  generateScreenID,
-  getPluginParams,
-  DocumentType,
-} = require("../../db/utils")
+const { getScreenParams, generateScreenID } = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { updateAppPackage } = require("./application")
 
 exports.fetch = async ctx => {
   const db = getAppDB()
@@ -39,48 +32,6 @@ exports.save = async ctx => {
 
   const response = await db.put(screen)
 
-  // Find any custom components being used
-  let pluginNames = []
-  let pluginAdded = false
-  findPlugins(screen.props, pluginNames)
-  if (pluginNames.length) {
-    const globalDB = getGlobalDB()
-    const pluginsResponse = await globalDB.allDocs(
-      getPluginParams(null, {
-        include_docs: true,
-      })
-    )
-    const requiredPlugins = pluginsResponse.rows
-      .map(row => row.doc)
-      .filter(plugin => {
-        return (
-          plugin.schema.type === "component" &&
-          pluginNames.includes(`plugin/${plugin.name}`)
-        )
-      })
-
-    // Update the app metadata
-    const application = await db.get(DocumentType.APP_METADATA)
-    let usedPlugins = application.usedPlugins || []
-
-    requiredPlugins.forEach(plugin => {
-      if (!usedPlugins.find(x => x._id === plugin._id)) {
-        pluginAdded = true
-        usedPlugins.push({
-          _id: plugin._id,
-          name: plugin.name,
-          version: plugin.version,
-          jsUrl: plugin.jsUrl,
-          hash: plugin.hash,
-        })
-      }
-    })
-
-    if (pluginAdded) {
-      await updateAppPackage({ usedPlugins }, ctx.appId)
-    }
-  }
-
   if (eventFn) {
     await eventFn(screen)
   }
@@ -89,7 +40,6 @@ exports.save = async ctx => {
     ...screen,
     _id: response.id,
     _rev: response.rev,
-    pluginAdded,
   }
 }
 
@@ -106,18 +56,3 @@ exports.destroy = async ctx => {
   }
   ctx.status = 200
 }
-
-const findPlugins = (component, foundPlugins) => {
-  if (!component) {
-    return
-  }
-  if (component._component.startsWith("plugin")) {
-    if (!foundPlugins.includes(component._component)) {
-      foundPlugins.push(component._component)
-    }
-  }
-  if (!component._children || !component._children.length) {
-    return
-  }
-  component._children.forEach(child => findPlugins(child, foundPlugins))
-}

package/src/api/controllers/static/index.ts

@@ -113,7 +113,6 @@ export const serveApp = async function (ctx: any) {
       production: env.isProd(),
       appId,
       clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
-      usedPlugins: appInfo.usedPlugins,
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -3,7 +3,6 @@
   export let favicon = ""
 
   export let clientLibPath
-  export let usedPlugins
 </script>
 
 <svelte:head>
@@ -84,15 +83,6 @@
   </script>
   <script type="application/javascript" src={clientLibPath}>
   </script>
-  <!-- Custom components need inserted after the core client library -->
-  <!-- But before loadBudibase is called -->
-  {#if usedPlugins?.length}
-    {#each usedPlugins as plugin}
-      <script
-        type="application/javascript"
-        src={`/plugins/${plugin.jsUrl}`}></script>
-    {/each}
-  {/if}
   <script type="application/javascript">
     if (window.loadBudibase) {
       window.loadBudibase()

package/src/api/controllers/static/templates/preview.hbs

@@ -66,9 +66,7 @@
         customTheme,
         previewDevice,
         navigation,
-        hiddenComponentIds,
-        usedPlugins,
-        location
+        hiddenComponentIds
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -83,8 +81,6 @@
       window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
-      window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
-      window["##BUDIBASE_LOCATION##"] = location
 
       // Initialise app
       try {

package/src/api/routes/index.ts

@@ -24,7 +24,6 @@ import metadataRoutes from "./metadata"
 import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
-import pluginRoutes from "./plugin"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
@@ -58,5 +57,4 @@ export const mainRoutes = [
   tableRoutes,
   rowRoutes,
   migrationRoutes,
-  pluginRoutes,
 ]

package/src/api/routes/tests/query.spec.js

@@ -231,6 +231,20 @@ describe("/queries", () => {
         url: `/api/queries/preview`,
       })
     })
+
+    it("should fail with invalid integration type", async () => {
+      const { datasource } = await createInvalidIntegration()
+      await request
+        .post(`/api/queries/preview`)
+        .send({
+          datasourceId: datasource._id,
+          parameters: {},
+          fields: {},
+          queryVerb: "read",
+        })
+        .set(config.defaultHeaders())
+        .expect(400)
+    })
   })
 
   describe("execute", () => {
@@ -247,14 +261,17 @@ describe("/queries", () => {
     })
 
     it("should fail with invalid integration type", async () => {
-      let error
-      try {
-        await createInvalidIntegration()
-      } catch (err) {
-        error = err
-      }
-      expect(error).toBeDefined()
-      expect(error.message).toBe("No datasource implementation found.")
+      const { query, datasource } = await createInvalidIntegration()
+      await request
+        .post(`/api/queries/${query._id}`)
+        .send({
+          datasourceId: datasource._id,
+          parameters: {},
+          fields: {},
+          queryVerb: "read",
+        })
+        .set(config.defaultHeaders())
+        .expect(400)
     })
   })
 

package/src/app.ts

@@ -26,13 +26,10 @@ const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
 const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
-const fs = require("fs")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
-import { watch } from "./watch"
-import { initialise as initialiseWebsockets } from "./websocket"
 
 const app = new Koa()
 
@@ -77,7 +74,6 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(server)
 
 let shuttingDown = false,
   errCode = 0
@@ -143,16 +139,6 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
   // check for version updates
   await installation.checkInstallVersion()
 

package/src/db/utils.js

@@ -42,7 +42,6 @@ const DocumentType = {
   MEM_VIEW: "view",
   USER_FLAG: "flag",
   AUTOMATION_METADATA: "meta_au",
-  PLUGIN: "plg",
 }
 
 const InternalTables = {
@@ -371,10 +370,6 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentType.MEM_VIEW, null, otherProps)
 }
 
-exports.generatePluginID = name => {
-  return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
-}
-
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */
@@ -384,10 +379,3 @@ exports.getMultiIDParams = ids => {
     include_docs: true,
   }
 }
-
-/**
- * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
- */
-exports.getPluginParams = (pluginId = null, otherProps = {}) => {
-  return getDocParams(DocumentType.PLUGIN, pluginId, otherProps)
-}

package/src/environment.js

@@ -37,8 +37,8 @@ function parseIntSafe(number) {
 let inThread = false
 
 module.exports = {
-  // important
-  PORT: process.env.PORT || process.env.APP_PORT,
+  // important - prefer app port to generic port
+  PORT: process.env.APP_PORT || process.env.PORT,
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
@@ -74,7 +74,6 @@ module.exports = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
-  PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,