@budibase/server 0.9.184 → 0.9.185-alpha.11

package/dist/utilities/csvParser.js

@@ -56,7 +56,7 @@ function parse(csvString, parsers) {
         });
         result.subscribe(row => {
             // For each CSV row parse all the columns that need parsed
-            for (let key in parsers) {
+            for (let key of Object.keys(parsers)) {
                 if (!schema[key] || schema[key].success) {
                     // get the validator for the column type
                     const validator = VALIDATORS[parsers[key].type];
@@ -79,15 +79,55 @@ function parse(csvString, parsers) {
         });
     });
 }
-function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+    if (!schema) {
+        return schema;
+    }
+    const finalSchema = {};
+    const schemaKeyMap = {};
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+    for (let [key, field] of Object.entries(existingTable.schema)) {
+        const lcKey = key.toLowerCase();
+        const foundKey = schemaKeyMap[lcKey];
+        if (foundKey) {
+            finalSchema[key] = schema[foundKey];
+            finalSchema[key].type = field.type;
+        }
+    }
+    return finalSchema;
+}
+function transform({ schema, csvString, existingTable }) {
     return __awaiter(this, void 0, void 0, function* () {
         const colParser = {};
-        for (let key in schema) {
+        // make sure the table has all the columns required for import
+        if (existingTable) {
+            schema = updateSchema({ schema, existingTable });
+        }
+        for (let key of Object.keys(schema)) {
             colParser[key] = PARSERS[schema[key].type] || schema[key].type;
         }
         try {
-            const json = yield csv({ colParser }).fromString(csvString);
-            return json;
+            const data = yield csv({ colParser }).fromString(csvString);
+            const schemaKeyMap = {};
+            Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+            for (let element of data) {
+                if (!data) {
+                    continue;
+                }
+                for (let key of Object.keys(element)) {
+                    const mappedKey = schemaKeyMap[key.toLowerCase()];
+                    // isn't a column in the table, remove it
+                    if (mappedKey == null) {
+                        delete element[key];
+                    }
+                    // casing is different, fix it in row
+                    else if (key !== mappedKey) {
+                        element[mappedKey] = element[key];
+                        delete element[key];
+                    }
+                }
+            }
+            return data;
         }
         catch (err) {
             console.error(`Error transforming CSV to JSON for data import`, err);
@@ -98,4 +138,5 @@ function transform({ schema, csvString }) {
 module.exports = {
     parse,
     transform,
+    updateSchema,
 };

package/dist/utilities/global.js

@@ -50,9 +50,12 @@ exports.getCachedSelf = (ctx, appId) => __awaiter(void 0, void 0, void 0, functi
     const user = yield userCache.getUser(ctx.user._id);
     return processUser(appId, user);
 });
-exports.getGlobalUser = (appId, userId) => __awaiter(void 0, void 0, void 0, function* () {
+exports.getRawGlobalUser = (userId) => __awaiter(void 0, void 0, void 0, function* () {
     const db = getGlobalDB();
-    let user = yield db.get(getGlobalIDFromUserMetadataID(userId));
+    return db.get(getGlobalIDFromUserMetadataID(userId));
+});
+exports.getGlobalUser = (appId, userId) => __awaiter(void 0, void 0, void 0, function* () {
+    let user = yield exports.getRawGlobalUser(userId);
     return processUser(appId, user);
 });
 exports.getGlobalUsers = (appId = null, users = null) => __awaiter(void 0, void 0, void 0, function* () {
@@ -71,6 +74,7 @@ exports.getGlobalUsers = (appId = null, users = null) => __awaiter(void 0, void
         .filter(user => user != null)
         .map(user => {
         delete user.password;
+        delete user.forceResetPassword;
         return user;
     });
     if (!appId) {

package/dist/utilities/index.js

@@ -52,12 +52,18 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = appId => {
+exports.clientLibraryPath = (appId, version) => {
     if (env.isProd()) {
-        return `${exports.objectStoreUrl()}/${sanitizeKey(appId)}/budibase-client.js`;
+        let url = `${exports.objectStoreUrl()}/${sanitizeKey(appId)}/budibase-client.js`;
+        // append app version to bust the cache
+        if (version) {
+            url += `?v=${version}`;
+        }
+        return url;
     }
     else {
         return `/api/assets/client`;
@@ -126,3 +132,13 @@ exports.stringToReadStream = string => {
         },
     });
 };
+exports.doesDatabaseExist = (dbName) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const db = new CouchDB(dbName, { skip_setup: true });
+        const info = yield db.info();
+        return info && !info.error;
+    }
+    catch (err) {
+        return false;
+    }
+});

package/dist/utilities/queue/inMemoryQueue.js

@@ -93,5 +93,13 @@ class InMemoryQueue {
     getRepeatableJobs() {
         return [];
     }
+    /**
+     * Implemented for tests
+     */
+    clean() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return [];
+        });
+    }
 }
 module.exports = InMemoryQueue;

package/dist/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 "use strict";
 const fetch = require("node-fetch");
 const { VM, VMScript } = require("vm2");
+const JS_TIMEOUT_MS = 1000;
 class ScriptRunner {
     constructor(script, context) {
         const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
-        this.vm = new VM();
+        this.vm = new VM({
+            timeout: JS_TIMEOUT_MS,
+        });
         this.results = { out: "" };
         this.vm.setGlobals(context);
         this.vm.setGlobal("fetch", fetch);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.184",
+  "version": "0.9.185-alpha.11",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,11 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.184",
-    "@budibase/client": "^0.9.184",
-    "@budibase/string-templates": "^0.9.184",
+    "@budibase/auth": "^0.9.185-alpha.11",
+    "@budibase/client": "^0.9.185-alpha.11",
+    "@budibase/string-templates": "^0.9.185-alpha.11",
+    "@bull-board/api": "^3.7.0",
+    "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
     "aws-sdk": "^2.767.0",
     "bcryptjs": "2.4.3",
     "bull": "^3.22.4",
-    "bull-board": "^2.0.1",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
     "dotenv": "8.2.0",
@@ -120,6 +121,7 @@
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
+    "worker-farm": "^1.7.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -138,7 +140,6 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
-    "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
     "prettier": "^2.3.1",
@@ -149,5 +150,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "4db512d87ec1a8cb90196a5428ad48e4411e1985"
+  "gitHead": "d6018c2c9cb114e5daa8e9e12f5ccc461e0f6f0b"
 }

package/scripts/integrations/mssql/data/Dockerfile

@@ -0,0 +1,9 @@
+FROM mcr.microsoft.com/mssql/server
+
+ENV ACCEPT_EULA=Y
+ENV SA_PASSWORD=Passw0rd
+
+COPY ./data /
+
+ENTRYPOINT [ "/bin/bash", "entrypoint.sh" ]
+CMD [ "/opt/mssql/bin/sqlservr" ]

package/scripts/integrations/mssql/data/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then
+  # If this is the container's first run, initialize the application database
+  if [ ! -f /tmp/app-initialized ]; then
+    # Initialize the application database asynchronously in a background process. This allows a) the SQL Server process to be the main process in the container, which allows graceful shutdown and other goodies, and b) us to only start the SQL Server process once, as opposed to starting, stopping, then starting it again.
+    function initialize_app_database() {
+      # Wait a bit for SQL Server to start. SQL Server's process doesn't provide a clever way to check if it's up or not, and it needs to be up before we can import the application database
+      sleep 30s
+
+      echo "RUNNING BUDIBASE SETUP"
+
+      #run the setup script to create the DB and the schema in the DB
+      /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql
+
+      # Note that the container has been initialized so future starts won't wipe changes to the data
+      touch /tmp/app-initialized
+    }
+    initialize_app_database &
+  fi
+fi
+
+exec "$@"

package/scripts/integrations/mssql/data/setup.sql

@@ -0,0 +1,54 @@
+USE master;
+  
+IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL  
+  DROP TABLE products;  
+GO  
+CREATE TABLE products 
+(  
+  id int IDENTITY(1,1),  
+  name varchar (20),  
+  description varchar(30),
+  CONSTRAINT pk_products PRIMARY KEY NONCLUSTERED (id)
+);
+
+IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL
+  DROP TABLE tasks;
+GO
+CREATE TABLE tasks
+(
+  taskid int IDENTITY(1,1),
+  taskname varchar (20),
+  productid int,
+  CONSTRAINT pk_tasks PRIMARY KEY NONCLUSTERED (taskid),
+  CONSTRAINT fk_products FOREIGN KEY (productid) REFERENCES products (id),
+);
+
+IF OBJECT_ID ('dbo.people', 'U') IS NOT NULL
+  DROP TABLE people;
+GO
+CREATE TABLE people
+(
+  name varchar(30),
+  age varchar(20),
+  CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age)
+);
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Bananas', 'Fruit thing');  
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Meat', 'Animal thing');
+
+INSERT tasks
+  (taskname, productid)
+VALUES
+  ('Processing', 1);
+
+INSERT people
+  (name, age)
+VALUES
+  ('Bob', '30');

package/scripts/integrations/mssql/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3.8"
+services:
+  # password: Passw0rd
+  # user: sa
+  # database: master
+  mssql:
+    image: bb/mssql
+    build:
+      context: .
+      dockerfile: data/Dockerfile
+    ports:
+      - "1433:1433"

package/scripts/integrations/mssql/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

package/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -44,6 +44,9 @@ const {
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -197,7 +200,7 @@ exports.fetchAppPackage = async ctx => {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId),
+    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
   }
 }
 
@@ -253,6 +256,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }
@@ -316,8 +320,12 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -328,6 +336,8 @@ exports.sync = async (ctx, next) => {
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
+
+  // replicate prod to dev
   const prodAppId = getDeployedAppID(appId)
 
   try {
@@ -357,6 +367,10 @@ exports.sync = async (ctx, next) => {
   } catch (err) {
     error = err
   }
+
+  // sync the users
+  await syncGlobalUsers(appId)
+
   if (error) {
     ctx.throw(400, error)
   } else {
@@ -380,7 +394,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {

package/src/api/controllers/auth.js

@@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => {
   }
 
   const user = await getFullUser(ctx, userId)
+  // this shouldn't be returned by the app self
+  delete user.roles
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => {
       // user has a role of some sort, return them
       else if (err.status === 404) {
         const metadata = {
+          ...user,
           _id: userId,
         }
         const dbResp = await db.put(metadata)

package/src/api/controllers/datasource.js

@@ -7,8 +7,9 @@ const {
   BudibaseInternalDB,
   getTableParams,
 } = require("../../db/utils")
+const { BuildSchemaErrors } = require("../../constants")
 const { integrations } = require("../../integrations")
-const { makeExternalQuery } = require("./row/utils")
+const { getDatasourceAndQuery } = require("./row/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -43,13 +44,17 @@ exports.buildSchemaFromDb = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasource = await db.get(ctx.params.datasourceId)
 
-  const tables = await buildSchemaHelper(datasource)
+  const { tables, error } = await buildSchemaHelper(datasource)
   datasource.entities = tables
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
-  ctx.body = datasource
+  const response = { datasource }
+  if (error) {
+    response.error = error
+  }
+  ctx.body = response
 }
 
 exports.update = async function (ctx) {
@@ -71,7 +76,7 @@ exports.update = async function (ctx) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  ctx.body = { datasource }
 }
 
 exports.save = async function (ctx) {
@@ -85,13 +90,15 @@ exports.save = async function (ctx) {
     ...ctx.request.body.datasource,
   }
 
+  let schemaError = null
   if (fetchSchema) {
-    let tables = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildSchemaHelper(datasource)
+    schemaError = error
     datasource.entities = tables
   }
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
@@ -101,17 +108,27 @@ exports.save = async function (ctx) {
     }
   }
 
-  ctx.status = 200
-  ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  const response = { datasource }
+  if (schemaError) {
+    response.error = schemaError
+  }
+  ctx.body = response
 }
 
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   // Delete all queries for the datasource
-  const rows = await db.allDocs(getQueryParams(ctx.params.datasourceId, null))
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  const queries = await db.allDocs(
+    getQueryParams(ctx.params.datasourceId, null)
+  )
+  await db.bulkDocs(
+    queries.rows.map(row => ({
+      _id: row.id,
+      _rev: row.value.rev,
+      _deleted: true,
+    }))
+  )
 
   // delete the datasource
   await db.remove(ctx.params.datasourceId, ctx.params.revId)
@@ -129,7 +146,7 @@ exports.find = async function (ctx) {
 exports.query = async function (ctx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await makeExternalQuery(ctx.appId, queryJson)
+    ctx.body = await getDatasourceAndQuery(ctx.appId, queryJson)
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -143,5 +160,28 @@ const buildSchemaHelper = async datasource => {
   await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
-  return connector.tables
+  // make sure they all have a display name selected
+  for (let entity of Object.values(datasource.entities)) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKeyTables = Object.entries(errors)
+      .filter(entry => entry[1] === BuildSchemaErrors.NO_KEY)
+      .map(([name]) => name)
+    error = `No primary key constraint found for the following: ${noKeyTables.join(
+      ", "
+    )}`
+  }
+  return { tables: connector.tables, error }
 }

package/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

package/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -24,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {
@@ -106,6 +108,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }