@budibase/server 0.9.185-alpha.2 → 0.9.185-alpha.20

package/dist/utilities/index.js

@@ -52,6 +52,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */

package/dist/utilities/queue/inMemoryQueue.js

@@ -93,5 +93,13 @@ class InMemoryQueue {
     getRepeatableJobs() {
         return [];
     }
+    /**
+     * Implemented for tests
+     */
+    clean() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return [];
+        });
+    }
 }
 module.exports = InMemoryQueue;

package/dist/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 "use strict";
 const fetch = require("node-fetch");
 const { VM, VMScript } = require("vm2");
+const JS_TIMEOUT_MS = 1000;
 class ScriptRunner {
     constructor(script, context) {
         const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
-        this.vm = new VM();
+        this.vm = new VM({
+            timeout: JS_TIMEOUT_MS,
+        });
         this.results = { out: "" };
         this.vm.setGlobals(context);
         this.vm.setGlobal("fetch", fetch);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.185-alpha.2",
+  "version": "0.9.185-alpha.20",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,11 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.185-alpha.2",
-    "@budibase/client": "^0.9.185-alpha.2",
-    "@budibase/string-templates": "^0.9.185-alpha.2",
+    "@budibase/auth": "^0.9.185-alpha.20",
+    "@budibase/client": "^0.9.185-alpha.20",
+    "@budibase/string-templates": "^0.9.185-alpha.20",
+    "@bull-board/api": "^3.7.0",
+    "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
     "aws-sdk": "^2.767.0",
     "bcryptjs": "2.4.3",
     "bull": "^3.22.4",
-    "bull-board": "^2.0.1",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
     "dotenv": "8.2.0",
@@ -120,6 +121,7 @@
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
+    "worker-farm": "^1.7.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -133,12 +135,12 @@
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
     "@types/node": "^15.12.4",
+    "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
-    "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
     "prettier": "^2.3.1",
@@ -149,5 +151,8 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "5bb8900cdd92acf802ce2529ccabbeaa745f4b37"
+  "optionalDependencies": {
+    "oracledb": "^5.3.0"
+  },
+  "gitHead": "5f40c6e234ea415c93ced4a2d13a0c2194be94a2"
 }

package/scripts/dev/manage.js

@@ -50,6 +50,7 @@ async function init() {
       SELF_HOSTED: 1,
       DISABLE_ACCOUNT_PORTAL: "",
       MULTI_TENANCY: "",
+      DISABLE_THREADING: 1,
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/scripts/integrations/oracle/docker-compose.yml

@@ -0,0 +1,19 @@
+# For more information see:
+# https://container-registry.oracle.com/
+# - Database > Express
+version: "3.8"
+services:
+  db:
+    container_name: oracle-xe
+    platform: linux/x86_64
+    image: container-registry.oracle.com/database/express:18.4.0-xe
+    environment:
+      ORACLE_PWD: oracle
+    ports:
+      - 1521:1521
+      - 5500:5500
+    volumes:
+      - oracle_data:/opt/oracle/oradata
+
+volumes:
+ oracle_data:

package/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+
+# Must be root to continue
+if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
+
+# Allow for re-runs
+rm -rf /opt/oracle
+
+echo "Installing oracle instant client"
+
+# copy and unzip package
+mkdir -p /opt/oracle
+cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
+cd /opt/oracle
+unzip -qq basiclite-21.4.zip -d .
+rm *.zip
+mv instantclient* instantclient
+
+# update runtime link path
+sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
+ldconfig /etc/ld.so.conf.d
+
+echo "Installation complete"

package/scripts/integrations/oracle/oracle.md

@@ -0,0 +1,90 @@
+# Installation
+
+## Database
+
+**Important**
+- Oracle database is supported only on **x86-64 architecture**
+- Oracle database is **not supported on Mac ARM architecture** (either via docker or linux virtualization)
+
+To install oracle express edition simply run `docker-compose up`
+
+- A single instance pluggable database (PDB) will be created named `xepdb`
+- The default password is configured in the compose file as `oracle`
+  - The `system`, `sys` and `pdbadmin` users all share this password
+
+## Instant Client 
+
+Before oracle can be connected to from nodejs, the oracle client must be installed.
+For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html
+
+**Important**
+- Oracle client is supported only on **x86-64 architecture**
+- Oracle client is **not supported on Mac ARM architecture** 
+
+### Linux
+Run the provided install script for linux from the `server` root path:
+
+```bash
+sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
+```
+For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst
+
+### Mac 
+**This has not yet been tested**
+
+See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst
+
+# Management
+To connect to oracle sql command line:
+
+```bash
+docker exec -it oracle-xe sqlplus -l system/oracle@localhost/xepdb1
+```
+
+To create a new schema (where a user is the same as a schema in oracle) named `sales`:
+
+```sql
+define USERNAME = sales
+
+create user &USERNAME;
+
+alter user &USERNAME
+    default tablespace users
+    temporary tablespace temp
+    quota unlimited on users;
+
+grant create session,
+    create view,
+    create sequence,
+    create procedure,
+    create table,
+    create trigger,
+    create type,
+    create materialized view
+    to &USERNAME;
+```
+
+To set the password for the sales schema use: 
+
+```sql
+define USERNAME = sales
+define PASSWORD = sales
+
+alter user &USERNAME identified by &PASSWORD;
+```
+
+As before the database schema can now be connected to using:
+```bash
+docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1
+```
+
+## HR Schema
+
+The `HR` schema is populated with dummy data by default in oracle for testing purposes. 
+To connect to the HR schema first update the user password and unlock the account by performing
+```sql
+ALTER USER hr ACCOUNT UNLOCK;
+ALTER USER hr IDENTIFIED BY hr
+```
+You should now be able to connect to the hr schema using the credentials hr/hr
+

package/scripts/integrations/postgres/docker-compose.yml

@@ -11,7 +11,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      #- pg_data:/var/lib/postgresql/data/
+      - pg_data:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
@@ -24,5 +24,5 @@ services:
     ports:
       - "5050:80"
 
-#volumes:
-#  pg_data:
+volumes:
+ pg_data:

package/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -45,6 +45,8 @@ const {
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
 const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -254,6 +256,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }
@@ -317,8 +320,12 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -387,7 +394,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   BudibaseInternalDB,
   getTableParams,
 } = require("../../db/utils")
-const { BuildSchemaErrors } = require("../../constants")
+const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 
@@ -119,8 +119,16 @@ exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   // Delete all queries for the datasource
-  const rows = await db.allDocs(getQueryParams(ctx.params.datasourceId, null))
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  const queries = await db.allDocs(
+    getQueryParams(ctx.params.datasourceId, null)
+  )
+  await db.bulkDocs(
+    queries.rows.map(row => ({
+      _id: row.id,
+      _rev: row.value.rev,
+      _deleted: true,
+    }))
+  )
 
   // delete the datasource
   await db.remove(ctx.params.datasourceId, ctx.params.revId)
@@ -144,6 +152,23 @@ exports.query = async function (ctx) {
   }
 }
 
+function getErrorTables(errors, errorType) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error, newError, tables) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
 const buildSchemaHelper = async datasource => {
   const Connector = integrations[datasource.source]
 
@@ -168,12 +193,23 @@ const buildSchemaHelper = async datasource => {
   const errors = connector.schemaErrors
   let error = null
   if (errors && Object.keys(errors).length > 0) {
-    const noKeyTables = Object.entries(errors)
-      .filter(entry => entry[1] === BuildSchemaErrors.NO_KEY)
-      .map(([name]) => name)
-    error = `No primary key constraint found for the following: ${noKeyTables.join(
-      ", "
-    )}`
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
   }
   return { tables: connector.tables, error }
 }

package/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

package/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -24,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {
@@ -106,6 +108,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }

package/src/api/controllers/query.js

@@ -1,10 +1,11 @@
 const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
 const { generateQueryID, getQueryParams } = require("../../db/utils")
-const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
-const ScriptRunner = require("../../utilities/scriptRunner")
+const { Thread, ThreadType } = require("../../threads")
+
+const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -18,47 +19,6 @@ function enrichQueries(input) {
   return wasArray ? queries : queries[0]
 }
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
-  }
-  return resp
-}
-
-async function runAndTransform(
-  integration,
-  queryVerb,
-  enrichedQuery,
-  transformer
-) {
-  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
-
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
-
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
-
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
-  }
-
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
-
-  return { rows, keys }
-}
-
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -143,30 +103,23 @@ exports.preview = async function (ctx) {
 
   const datasource = await db.get(ctx.request.body.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-  const integration = new Integration(datasource.config)
 
-  const { rows, keys } = await runAndTransform(
-    integration,
-    queryVerb,
-    enrichedQuery,
-    transformer
-  )
+  try {
+    const { rows, keys } = await Runner.run({
+      datasource,
+      queryVerb,
+      query: enrichedQuery,
+      transformer,
+    })
 
-  ctx.body = {
-    rows,
-    schemaFields: [...new Set(keys)],
-  }
-  // cleanup
-  if (integration.end) {
-    integration.end()
+    ctx.body = {
+      rows,
+      schemaFields: [...new Set(keys)],
+    }
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 
@@ -176,29 +129,22 @@ exports.execute = async function (ctx) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const enrichedQuery = await enrichQueryFields(
     query.fields,
     ctx.request.body.parameters
   )
-  const integration = new Integration(datasource.config)
 
   // call the relevant CRUD method on the integration class
-  const { rows } = await runAndTransform(
-    integration,
-    query.queryVerb,
-    enrichedQuery,
-    query.transformer
-  )
-  ctx.body = rows
-  // cleanup
-  if (integration.end) {
-    integration.end()
+  try {
+    const { rows } = await Runner.run({
+      datasource,
+      queryVerb: query.queryVerb,
+      query: enrichedQuery,
+      transformer: query.transformer,
+    })
+    ctx.body = rows
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 

package/src/api/controllers/table/external.js

@@ -161,9 +161,13 @@ function isRelationshipSetup(column) {
 exports.save = async function (ctx) {
   const appId = ctx.appId
   const table = ctx.request.body
-  // can't do this
+  // can't do this right now
   delete table.dataImport
   const datasourceId = getDatasourceId(ctx.request.body)
+  // table doesn't exist already, note that it is created
+  if (!table._id) {
+    table.created = true
+  }
   let tableToSave = {
     type: "table",
     _id: buildExternalTableId(datasourceId, table.name),
@@ -265,6 +269,9 @@ exports.save = async function (ctx) {
 exports.destroy = async function (ctx) {
   const appId = ctx.appId
   const tableToDelete = await getTable(appId, ctx.params.tableId)
+  if (!tableToDelete || !tableToDelete.created) {
+    ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
+  }
   const datasourceId = getDatasourceId(tableToDelete)
 
   const db = new CouchDB(appId)

package/src/api/routes/query.js

@@ -66,6 +66,7 @@ router
   )
   .get(
     "/api/queries/:queryId",
+    paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter