@budibase/server 0.9.186 → 0.9.190-alpha.0

package/dist/utilities/csvParser.js

@@ -56,7 +56,7 @@ function parse(csvString, parsers) {
         });
         result.subscribe(row => {
             // For each CSV row parse all the columns that need parsed
-            for (let key in parsers) {
+            for (let key of Object.keys(parsers)) {
                 if (!schema[key] || schema[key].success) {
                     // get the validator for the column type
                     const validator = VALIDATORS[parsers[key].type];
@@ -79,15 +79,55 @@ function parse(csvString, parsers) {
         });
     });
 }
-function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+    if (!schema) {
+        return schema;
+    }
+    const finalSchema = {};
+    const schemaKeyMap = {};
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+    for (let [key, field] of Object.entries(existingTable.schema)) {
+        const lcKey = key.toLowerCase();
+        const foundKey = schemaKeyMap[lcKey];
+        if (foundKey) {
+            finalSchema[key] = schema[foundKey];
+            finalSchema[key].type = field.type;
+        }
+    }
+    return finalSchema;
+}
+function transform({ schema, csvString, existingTable }) {
     return __awaiter(this, void 0, void 0, function* () {
         const colParser = {};
-        for (let key in schema) {
+        // make sure the table has all the columns required for import
+        if (existingTable) {
+            schema = updateSchema({ schema, existingTable });
+        }
+        for (let key of Object.keys(schema)) {
             colParser[key] = PARSERS[schema[key].type] || schema[key].type;
         }
         try {
-            const json = yield csv({ colParser }).fromString(csvString);
-            return json;
+            const data = yield csv({ colParser }).fromString(csvString);
+            const schemaKeyMap = {};
+            Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+            for (let element of data) {
+                if (!data) {
+                    continue;
+                }
+                for (let key of Object.keys(element)) {
+                    const mappedKey = schemaKeyMap[key.toLowerCase()];
+                    // isn't a column in the table, remove it
+                    if (mappedKey == null) {
+                        delete element[key];
+                    }
+                    // casing is different, fix it in row
+                    else if (key !== mappedKey) {
+                        element[mappedKey] = element[key];
+                        delete element[key];
+                    }
+                }
+            }
+            return data;
         }
         catch (err) {
             console.error(`Error transforming CSV to JSON for data import`, err);
@@ -98,4 +138,5 @@ function transform({ schema, csvString }) {
 module.exports = {
     parse,
     transform,
+    updateSchema,
 };

package/dist/utilities/global.js

@@ -50,9 +50,12 @@ exports.getCachedSelf = (ctx, appId) => __awaiter(void 0, void 0, void 0, functi
     const user = yield userCache.getUser(ctx.user._id);
     return processUser(appId, user);
 });
-exports.getGlobalUser = (appId, userId) => __awaiter(void 0, void 0, void 0, function* () {
+exports.getRawGlobalUser = (userId) => __awaiter(void 0, void 0, void 0, function* () {
     const db = getGlobalDB();
-    let user = yield db.get(getGlobalIDFromUserMetadataID(userId));
+    return db.get(getGlobalIDFromUserMetadataID(userId));
+});
+exports.getGlobalUser = (appId, userId) => __awaiter(void 0, void 0, void 0, function* () {
+    let user = yield exports.getRawGlobalUser(userId);
     return processUser(appId, user);
 });
 exports.getGlobalUsers = (appId = null, users = null) => __awaiter(void 0, void 0, void 0, function* () {
@@ -71,6 +74,7 @@ exports.getGlobalUsers = (appId = null, users = null) => __awaiter(void 0, void
         .filter(user => user != null)
         .map(user => {
         delete user.password;
+        delete user.forceResetPassword;
         return user;
     });
     if (!appId) {

package/dist/utilities/index.js

@@ -52,6 +52,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
@@ -131,3 +132,13 @@ exports.stringToReadStream = string => {
         },
     });
 };
+exports.doesDatabaseExist = (dbName) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const db = new CouchDB(dbName, { skip_setup: true });
+        const info = yield db.info();
+        return info && !info.error;
+    }
+    catch (err) {
+        return false;
+    }
+});

package/dist/utilities/queue/inMemoryQueue.js

@@ -93,5 +93,13 @@ class InMemoryQueue {
     getRepeatableJobs() {
         return [];
     }
+    /**
+     * Implemented for tests
+     */
+    clean() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return [];
+        });
+    }
 }
 module.exports = InMemoryQueue;

package/dist/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 "use strict";
 const fetch = require("node-fetch");
 const { VM, VMScript } = require("vm2");
+const JS_TIMEOUT_MS = 1000;
 class ScriptRunner {
     constructor(script, context) {
         const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
-        this.vm = new VM();
+        this.vm = new VM({
+            timeout: JS_TIMEOUT_MS,
+        });
         this.results = { out: "" };
         this.vm.setGlobals(context);
         this.vm.setGlobal("fetch", fetch);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.186",
+  "version": "0.9.190-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -14,7 +14,7 @@
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service",
+    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -66,11 +66,13 @@
     "budibase"
   ],
   "author": "Budibase",
-  "license": "AGPL-3.0-or-later",
+  "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^0.9.186",
-    "@budibase/client": "^0.9.186",
-    "@budibase/string-templates": "^0.9.186",
+    "@budibase/auth": "^0.9.190-alpha.0",
+    "@budibase/client": "^0.9.190-alpha.0",
+    "@budibase/string-templates": "^0.9.190-alpha.0",
+    "@bull-board/api": "^3.7.0",
+    "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
     "aws-sdk": "^2.767.0",
     "bcryptjs": "2.4.3",
     "bull": "^3.22.4",
-    "bull-board": "^2.0.1",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
     "dotenv": "8.2.0",
@@ -120,6 +121,7 @@
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
+    "worker-farm": "^1.7.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -133,12 +135,12 @@
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
     "@types/node": "^15.12.4",
+    "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
-    "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
     "prettier": "^2.3.1",
@@ -149,5 +151,8 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "4385ee6fc2e2e2ec9738aa29ee9cae463c2de54b"
+  "optionalDependencies": {
+    "oracledb": "^5.3.0"
+  },
+  "gitHead": "58c1927f6a5dcbab760fbdb9d581b1ef1bb9dabf"
 }

package/scripts/dev/manage.js

@@ -50,6 +50,7 @@ async function init() {
       SELF_HOSTED: 1,
       DISABLE_ACCOUNT_PORTAL: "",
       MULTI_TENANCY: "",
+      DISABLE_THREADING: 1,
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/scripts/integrations/mssql/data/Dockerfile

@@ -0,0 +1,9 @@
+FROM mcr.microsoft.com/mssql/server
+
+ENV ACCEPT_EULA=Y
+ENV SA_PASSWORD=Passw0rd
+
+COPY ./data /
+
+ENTRYPOINT [ "/bin/bash", "entrypoint.sh" ]
+CMD [ "/opt/mssql/bin/sqlservr" ]

package/scripts/integrations/mssql/data/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then
+  # If this is the container's first run, initialize the application database
+  if [ ! -f /tmp/app-initialized ]; then
+    # Initialize the application database asynchronously in a background process. This allows a) the SQL Server process to be the main process in the container, which allows graceful shutdown and other goodies, and b) us to only start the SQL Server process once, as opposed to starting, stopping, then starting it again.
+    function initialize_app_database() {
+      # Wait a bit for SQL Server to start. SQL Server's process doesn't provide a clever way to check if it's up or not, and it needs to be up before we can import the application database
+      sleep 30s
+
+      echo "RUNNING BUDIBASE SETUP"
+
+      #run the setup script to create the DB and the schema in the DB
+      /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql
+
+      # Note that the container has been initialized so future starts won't wipe changes to the data
+      touch /tmp/app-initialized
+    }
+    initialize_app_database &
+  fi
+fi
+
+exec "$@"

package/scripts/integrations/mssql/data/setup.sql

@@ -0,0 +1,54 @@
+USE master;
+  
+IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL  
+  DROP TABLE products;  
+GO  
+CREATE TABLE products 
+(  
+  id int IDENTITY(1,1),  
+  name varchar (20),  
+  description varchar(30),
+  CONSTRAINT pk_products PRIMARY KEY NONCLUSTERED (id)
+);
+
+IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL
+  DROP TABLE tasks;
+GO
+CREATE TABLE tasks
+(
+  taskid int IDENTITY(1,1),
+  taskname varchar (20),
+  productid int,
+  CONSTRAINT pk_tasks PRIMARY KEY NONCLUSTERED (taskid),
+  CONSTRAINT fk_products FOREIGN KEY (productid) REFERENCES products (id),
+);
+
+IF OBJECT_ID ('dbo.people', 'U') IS NOT NULL
+  DROP TABLE people;
+GO
+CREATE TABLE people
+(
+  name varchar(30),
+  age varchar(20),
+  CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age)
+);
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Bananas', 'Fruit thing');  
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Meat', 'Animal thing');
+
+INSERT tasks
+  (taskname, productid)
+VALUES
+  ('Processing', 1);
+
+INSERT people
+  (name, age)
+VALUES
+  ('Bob', '30');

package/scripts/integrations/mssql/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3.8"
+services:
+  # password: Passw0rd
+  # user: sa
+  # database: master
+  mssql:
+    image: bb/mssql
+    build:
+      context: .
+      dockerfile: data/Dockerfile
+    ports:
+      - "1433:1433"

package/scripts/integrations/mssql/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

package/scripts/integrations/oracle/docker-compose.yml

@@ -0,0 +1,19 @@
+# For more information see:
+# https://container-registry.oracle.com/
+# - Database > Express
+version: "3.8"
+services:
+  db:
+    container_name: oracle-xe
+    platform: linux/x86_64
+    image: container-registry.oracle.com/database/express:18.4.0-xe
+    environment:
+      ORACLE_PWD: oracle
+    ports:
+      - 1521:1521
+      - 5500:5500
+    volumes:
+      - oracle_data:/opt/oracle/oradata
+
+volumes:
+ oracle_data:

package/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+
+# Must be root to continue
+if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
+
+# Allow for re-runs
+rm -rf /opt/oracle
+
+echo "Installing oracle instant client"
+
+# copy and unzip package
+mkdir -p /opt/oracle
+cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
+cd /opt/oracle
+unzip -qq basiclite-21.4.zip -d .
+rm *.zip
+mv instantclient* instantclient
+
+# update runtime link path
+sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
+ldconfig /etc/ld.so.conf.d
+
+echo "Installation complete"

package/scripts/integrations/oracle/oracle.md

@@ -0,0 +1,90 @@
+# Installation
+
+## Database
+
+**Important**
+- Oracle database is supported only on **x86-64 architecture**
+- Oracle database is **not supported on Mac ARM architecture** (either via docker or linux virtualization)
+
+To install oracle express edition simply run `docker-compose up`
+
+- A single instance pluggable database (PDB) will be created named `xepdb`
+- The default password is configured in the compose file as `oracle`
+  - The `system`, `sys` and `pdbadmin` users all share this password
+
+## Instant Client 
+
+Before oracle can be connected to from nodejs, the oracle client must be installed.
+For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html
+
+**Important**
+- Oracle client is supported only on **x86-64 architecture**
+- Oracle client is **not supported on Mac ARM architecture** 
+
+### Linux
+Run the provided install script for linux from the `server` root path:
+
+```bash
+sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
+```
+For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst
+
+### Mac 
+**This has not yet been tested**
+
+See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst
+
+# Management
+To connect to oracle sql command line:
+
+```bash
+docker exec -it oracle-xe sqlplus -l system/oracle@localhost/xepdb1
+```
+
+To create a new schema (where a user is the same as a schema in oracle) named `sales`:
+
+```sql
+define USERNAME = sales
+
+create user &USERNAME;
+
+alter user &USERNAME
+    default tablespace users
+    temporary tablespace temp
+    quota unlimited on users;
+
+grant create session,
+    create view,
+    create sequence,
+    create procedure,
+    create table,
+    create trigger,
+    create type,
+    create materialized view
+    to &USERNAME;
+```
+
+To set the password for the sales schema use: 
+
+```sql
+define USERNAME = sales
+define PASSWORD = sales
+
+alter user &USERNAME identified by &PASSWORD;
+```
+
+As before the database schema can now be connected to using:
+```bash
+docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1
+```
+
+## HR Schema
+
+The `HR` schema is populated with dummy data by default in oracle for testing purposes. 
+To connect to the HR schema first update the user password and unlock the account by performing
+```sql
+ALTER USER hr ACCOUNT UNLOCK;
+ALTER USER hr IDENTIFIED BY hr
+```
+You should now be able to connect to the hr schema using the credentials hr/hr
+

package/scripts/integrations/postgres/docker-compose.yml

@@ -11,7 +11,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      #- pg_data:/var/lib/postgresql/data/
+      - pg_data:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
@@ -24,5 +24,5 @@ services:
     ports:
       - "5050:80"
 
-#volumes:
-#  pg_data:
+volumes:
+ pg_data:

package/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -44,6 +44,9 @@ const {
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -253,6 +256,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }
@@ -316,8 +320,12 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -328,6 +336,8 @@ exports.sync = async (ctx, next) => {
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
+
+  // replicate prod to dev
   const prodAppId = getDeployedAppID(appId)
 
   try {
@@ -357,6 +367,10 @@ exports.sync = async (ctx, next) => {
   } catch (err) {
     error = err
   }
+
+  // sync the users
+  await syncGlobalUsers(appId)
+
   if (error) {
     ctx.throw(400, error)
   } else {
@@ -380,7 +394,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {

package/src/api/controllers/auth.js

@@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => {
   }
 
   const user = await getFullUser(ctx, userId)
+  // this shouldn't be returned by the app self
+  delete user.roles
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => {
       // user has a role of some sort, return them
       else if (err.status === 404) {
         const metadata = {
+          ...user,
           _id: userId,
         }
         const dbResp = await db.put(metadata)