@budibase/server 1.3.19-alpha.1 → 1.3.19

package/dist/utilities/fileSystem/index.js

@@ -14,18 +14,14 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
-const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
-const tar = require("tar");
-const fetch = require("node-fetch");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
-const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -100,12 +96,6 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
 exports.loadHandlebarsFile = path => {
     return fs.readFileSync(path, "utf8");
 };
-/**
- * Same as above just with a different name.
- */
-exports.loadJSFile = (directory, name) => {
-    return fs.readFileSync(join(directory, name), "utf8");
-};
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -304,104 +294,6 @@ exports.cleanup = appIds => {
         }
     }
 };
-const createTempFolder = item => {
-    const path = join(budibaseTempDir(), item);
-    try {
-        // remove old tmp directories automatically - don't combine
-        if (fs.existsSync(path)) {
-            fs.rmSync(path, { recursive: true, force: true });
-        }
-        fs.mkdirSync(path);
-    }
-    catch (err) {
-        throw new Error(`Path cannot be created: ${err.message}`);
-    }
-    return path;
-};
-exports.createTempFolder = createTempFolder;
-const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
-    yield tar.extract({
-        file: fromFilePath,
-        C: toPath,
-    });
-});
-exports.extractTarball = extractTarball;
-const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
-    let metadata = {};
-    try {
-        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
-        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
-        metadata.schema = JSON.parse(schema);
-        metadata.package = JSON.parse(pkg);
-        if (!metadata.package.name ||
-            !metadata.package.version ||
-            !metadata.package.description) {
-            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
-        }
-    }
-    catch (err) {
-        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
-    }
-    return { metadata, directory: path };
-});
-exports.getPluginMetadata = getPluginMetadata;
-exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!fs.existsSync(DATASOURCE_PATH)) {
-        fs.mkdirSync(DATASOURCE_PATH);
-    }
-    const filename = join(DATASOURCE_PATH, name);
-    const metadataName = `${filename}.bbmetadata`;
-    if (fs.existsSync(filename)) {
-        const currentHash = fs.readFileSync(metadataName, "utf8");
-        // if hash is the same return the file, otherwise remove it and re-download
-        if (currentHash === hash) {
-            return require(filename);
-        }
-        else {
-            console.log(`Updating plugin: ${name}`);
-            fs.unlinkSync(filename);
-        }
-    }
-    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
-    const response = yield fetch(fullUrl);
-    if (response.status === 200) {
-        const content = yield response.text();
-        fs.writeFileSync(filename, content);
-        fs.writeFileSync(metadataName, hash);
-        return require(filename);
-    }
-    else {
-        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
-    }
-});
-/**
- * Find for a file recursively from start path applying filter, return first match
- */
-exports.findFileRec = (startPath, filter) => {
-    if (!fs.existsSync(startPath)) {
-        return;
-    }
-    const files = fs.readdirSync(startPath);
-    for (let i = 0, len = files.length; i < len; i++) {
-        const filename = join(startPath, files[i]);
-        const stat = fs.lstatSync(filename);
-        if (stat.isDirectory()) {
-            return exports.findFileRec(filename, filter);
-        }
-        else if (filename.endsWith(filter)) {
-            return filename;
-        }
-    }
-};
-/**
- * Remove a folder which is not empty from the file system
- */
-exports.deleteFolderFileSystem = path => {
-    if (!fs.existsSync(path)) {
-        return;
-    }
-    fs.rmSync(path, { recursive: true, force: true });
-};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -409,6 +301,5 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
-exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.19-alpha.1",
+  "version": "1.3.19",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.3.19-alpha.1",
-    "@budibase/client": "1.3.19-alpha.1",
-    "@budibase/pro": "1.3.19-alpha.0",
-    "@budibase/string-templates": "1.3.19-alpha.1",
-    "@budibase/types": "1.3.19-alpha.1",
+    "@budibase/backend-core": "^1.3.19",
+    "@budibase/client": "^1.3.19",
+    "@budibase/pro": "1.3.18",
+    "@budibase/string-templates": "^1.3.19",
+    "@budibase/types": "^1.3.19",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -95,7 +95,6 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
-    "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
@@ -139,10 +138,8 @@
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
-    "socket.io": "^4.5.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
-    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -199,5 +196,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "9eb4995976ec00b40ba52df7d0bc357e8df6edce"
+  "gitHead": "1e621da16bad75ae6a2dfcd732e676cf0b56a2a4"
 }

package/scripts/dev/manage.js

@@ -58,7 +58,6 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
-      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/scripts/localdomain.js

@@ -17,8 +17,6 @@ const arg = process.argv.slice(2)[0]
  */
 updateDotEnv({
   ACCOUNT_PORTAL_URL:
-    arg === "enable"
-      ? "http://account.local.com:10001"
-      : "http://localhost:10001",
+    arg === "enable" ? "http://local.com:10001" : "http://localhost:10001",
   COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
 }).then(() => console.log("Updated worker!"))

package/specs/openapi.json

@@ -1327,7 +1327,7 @@
             "type": "string"
           },
           "datasourceId": {
-            "description": "The ID of the datasource the query belongs to.",
+            "description": "The ID of the data source the query belongs to.",
             "type": "string"
           },
           "parameters": {
@@ -1386,7 +1386,7 @@
                   "type": "string"
                 },
                 "datasourceId": {
-                  "description": "The ID of the datasource the query belongs to.",
+                  "description": "The ID of the data source the query belongs to.",
                   "type": "string"
                 },
                 "parameters": {
@@ -2289,7 +2289,7 @@
         },
         "responses": {
           "200": {
-            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external datasources.",
+            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external data sources.",
             "content": {
               "application/json": {
                 "schema": {

package/specs/openapi.yaml

@@ -1004,7 +1004,7 @@ components:
           description: The ID of the query.
           type: string
         datasourceId:
-          description: The ID of the datasource the query belongs to.
+          description: The ID of the data source the query belongs to.
           type: string
         parameters:
           description: The bindings which are required to perform this query.
@@ -1051,7 +1051,7 @@ components:
                 description: The ID of the query.
                 type: string
               datasourceId:
-                description: The ID of the datasource the query belongs to.
+                description: The ID of the data source the query belongs to.
                 type: string
               parameters:
                 description: The bindings which are required to perform this query.
@@ -1665,7 +1665,7 @@ paths:
       responses:
         "200":
           description: Returns the created table, including the ID which has been
-            generated for it. This can be internal or external datasources.
+            generated for it. This can be internal or external data sources.
           content:
             application/json:
               schema:

package/specs/resources/query.js

@@ -82,7 +82,7 @@ const querySchema = object(
       type: "string",
     },
     datasourceId: {
-      description: "The ID of the datasource the query belongs to.",
+      description: "The ID of the data source the query belongs to.",
       type: "string",
     },
     parameters: {

package/src/api/controllers/application.ts

@@ -472,6 +472,7 @@ const destroyApp = async (ctx: any) => {
   const result = await db.destroy()
 
   if (isUnpublish) {
+    await quotas.removePublishedApp()
     await events.app.unpublished(app)
   } else {
     await quotas.removeApp()
@@ -571,7 +572,7 @@ export const sync = async (ctx: any, next: any) => {
   }
 }
 
-export const updateAppPackage = async (appPackage: any, appId: any) => {
+const updateAppPackage = async (appPackage: any, appId: any) => {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/auth.js

@@ -4,15 +4,6 @@ const { getFullUser } = require("../../utilities/users")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
-/**
- * Add the attributes that are session based to the current user.
- */
-const addSessionAttributesToUser = ctx => {
-  if (ctx.user) {
-    ctx.body.license = ctx.user.license
-  }
-}
-
 exports.fetchSelf = async ctx => {
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
@@ -64,6 +55,4 @@ exports.fetchSelf = async ctx => {
   } else {
     ctx.body = user
   }
-
-  addSessionAttributesToUser(ctx)
 }

package/src/api/controllers/component.js

@@ -1,7 +1,6 @@
-const { DocumentType, getPluginParams } = require("../../db/utils")
+const { DocumentType } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   try {
@@ -11,6 +10,7 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
     let componentManifests = await Promise.all(
       app.componentLibraries.map(async library => {
         let manifest = await getComponentLibraryManifest(library)
+
         return {
           manifest,
           library,
@@ -31,25 +31,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
-
-    // Add custom components
-    const globalDB = getGlobalDB()
-    const response = await globalDB.allDocs(
-      getPluginParams(null, {
-        include_docs: true,
-      })
-    )
-    response.rows
-      .map(row => row.doc)
-      .filter(plugin => plugin.schema.type === "component")
-      .forEach(plugin => {
-        const fullComponentName = `plugin/${plugin.name}`
-        definitions[fullComponentName] = {
-          component: fullComponentName,
-          ...plugin.schema.schema,
-        }
-      })
-
     ctx.body = definitions
   } catch (err) {
     console.error(`component-definitions=failed`, err)

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { getIntegration } = require("../../integrations")
+const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -50,23 +50,9 @@ exports.fetch = async function (ctx) {
 exports.buildSchemaFromDb = async function (ctx) {
   const db = getAppDB()
   const datasource = await db.get(ctx.params.datasourceId)
-  const tablesFilter = ctx.request.body.tablesFilter
 
-  let { tables, error } = await buildSchemaHelper(datasource)
-  if (tablesFilter) {
-    if (!datasource.entities) {
-      datasource.entities = {}
-    }
-    for (let key in tables) {
-      if (
-        tablesFilter.some(filter => filter.toLowerCase() === key.toLowerCase())
-      ) {
-        datasource.entities[key] = tables[key]
-      }
-    }
-  } else {
-    datasource.entities = tables
-  }
+  const { tables, error } = await buildSchemaHelper(datasource)
+  datasource.entities = tables
 
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
@@ -128,7 +114,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = await getIntegration(datasource.source)
+    const source = integrations[datasource.source]
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -163,7 +149,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = await getIntegration(datasource.source)
+    const source = integrations[datasource.source]
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -232,14 +218,15 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = await getIntegration(datasource.source)
+  const Connector = integrations[datasource.source]
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)
+  datasource.entities = connector.tables
 
   // make sure they all have a display name selected
-  for (let entity of Object.values(datasource.entities ?? {})) {
+  for (let entity of Object.values(datasource.entities)) {
     if (entity.primaryDisplay) {
       continue
     }

package/src/api/controllers/deploy/index.ts

@@ -206,7 +206,12 @@ const _deployApp = async function (ctx: any) {
 
   console.log("Deploying app...")
 
-  let app = await deployApp(deployment)
+  let app
+  if (await isFirstDeploy()) {
+    app = await quotas.addPublishedApp(() => deployApp(deployment))
+  } else {
+    app = await deployApp(deployment)
+  }
 
   await events.app.published(app)
   ctx.body = deployment

package/src/api/controllers/integration.js

@@ -1,11 +1,12 @@
-const { getDefinitions } = require("../../integrations")
+const { cloneDeep } = require("lodash")
+const { definitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = await getDefinitions()
+  const defs = cloneDeep(definitions)
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -16,7 +17,6 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = defs[ctx.params.type]
+  ctx.body = definitions[ctx.params.type]
 }

package/src/api/controllers/row/external.js

@@ -53,7 +53,7 @@ exports.save = async ctx => {
 }
 
 exports.fetchView = async ctx => {
-  // there are no views in external datasources, shouldn't ever be called
+  // there are no views in external data sources, shouldn't ever be called
   // for now just fetch
   const split = ctx.params.viewName.split("all_")
   ctx.params.tableId = split[1] ? split[1] : split[0]

package/src/api/controllers/screen.js

@@ -1,14 +1,7 @@
-const {
-  getScreenParams,
-  generateScreenID,
-  getPluginParams,
-  DocumentType,
-} = require("../../db/utils")
+const { getScreenParams, generateScreenID } = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { updateAppPackage } = require("./application")
 
 exports.fetch = async ctx => {
   const db = getAppDB()
@@ -39,48 +32,6 @@ exports.save = async ctx => {
 
   const response = await db.put(screen)
 
-  // Find any custom components being used
-  let pluginNames = []
-  let pluginAdded = false
-  findPlugins(screen.props, pluginNames)
-  if (pluginNames.length) {
-    const globalDB = getGlobalDB()
-    const pluginsResponse = await globalDB.allDocs(
-      getPluginParams(null, {
-        include_docs: true,
-      })
-    )
-    const requiredPlugins = pluginsResponse.rows
-      .map(row => row.doc)
-      .filter(plugin => {
-        return (
-          plugin.schema.type === "component" &&
-          pluginNames.includes(`plugin/${plugin.name}`)
-        )
-      })
-
-    // Update the app metadata
-    const application = await db.get(DocumentType.APP_METADATA)
-    let usedPlugins = application.usedPlugins || []
-
-    requiredPlugins.forEach(plugin => {
-      if (!usedPlugins.find(x => x._id === plugin._id)) {
-        pluginAdded = true
-        usedPlugins.push({
-          _id: plugin._id,
-          name: plugin.name,
-          version: plugin.version,
-          jsUrl: plugin.jsUrl,
-          hash: plugin.hash,
-        })
-      }
-    })
-
-    if (pluginAdded) {
-      await updateAppPackage({ usedPlugins }, ctx.appId)
-    }
-  }
-
   if (eventFn) {
     await eventFn(screen)
   }
@@ -89,7 +40,6 @@ exports.save = async ctx => {
     ...screen,
     _id: response.id,
     _rev: response.rev,
-    pluginAdded,
   }
 }
 
@@ -106,18 +56,3 @@ exports.destroy = async ctx => {
   }
   ctx.status = 200
 }
-
-const findPlugins = (component, foundPlugins) => {
-  if (!component) {
-    return
-  }
-  if (component._component.startsWith("plugin")) {
-    if (!foundPlugins.includes(component._component)) {
-      foundPlugins.push(component._component)
-    }
-  }
-  if (!component._children || !component._children.length) {
-    return
-  }
-  component._children.forEach(child => findPlugins(child, foundPlugins))
-}

package/src/api/controllers/static/index.ts

@@ -18,6 +18,7 @@ const { DocumentType } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
+
 const fs = require("fs")
 const {
   downloadTarballDirect,
@@ -112,7 +113,6 @@ export const serveApp = async function (ctx: any) {
       production: env.isProd(),
       appId,
       clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
-      usedPlugins: appInfo.usedPlugins,
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -3,7 +3,6 @@
   export let favicon = ""
 
   export let clientLibPath
-  export let usedPlugins
 </script>
 
 <svelte:head>
@@ -84,15 +83,6 @@
   </script>
   <script type="application/javascript" src={clientLibPath}>
   </script>
-  <!-- Custom components need inserted after the core client library -->
-  <!-- But before loadBudibase is called -->
-  {#if usedPlugins?.length}
-    {#each usedPlugins as plugin}
-      <script
-        type="application/javascript"
-        src={`/plugins/${plugin.jsUrl}`}></script>
-    {/each}
-  {/if}
   <script type="application/javascript">
     if (window.loadBudibase) {
       window.loadBudibase()

package/src/api/controllers/static/templates/preview.hbs

@@ -66,9 +66,7 @@
         customTheme,
         previewDevice,
         navigation,
-        hiddenComponentIds,
-        usedPlugins,
-        location
+        hiddenComponentIds
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -83,8 +81,6 @@
       window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
-      window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
-      window["##BUDIBASE_LOCATION##"] = location
 
       // Initialise app
       try {

package/src/api/index.js

@@ -54,8 +54,8 @@ router
       noTenancyRequired: true,
     })
   )
-  .use(pro.licensing())
   .use(currentApp)
+  .use(pro.licensing())
   .use(auditLog)
 
 // error handling middleware

package/src/api/routes/application.ts

@@ -2,27 +2,16 @@ import Router from "@koa/router"
 import * as controller from "../controllers/application"
 import authorized from "../../middleware/authorized"
 import { BUILDER } from "@budibase/backend-core/permissions"
-import { applicationValidator } from "./utils/validators"
 
 const router = new Router()
 
 router
   .post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync)
-  .post(
-    "/api/applications",
-    authorized(BUILDER),
-    applicationValidator(),
-    controller.create
-  )
+  .post("/api/applications", authorized(BUILDER), controller.create)
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
-  .put(
-    "/api/applications/:appId",
-    authorized(BUILDER),
-    applicationValidator(),
-    controller.update
-  )
+  .put("/api/applications/:appId", authorized(BUILDER), controller.update)
   .post(
     "/api/applications/:appId/client/update",
     authorized(BUILDER),