@budibase/server 0.9.188 → 0.9.190-alpha.10

package/scripts/integrations/mssql/data/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then
+  # If this is the container's first run, initialize the application database
+  if [ ! -f /tmp/app-initialized ]; then
+    # Initialize the application database asynchronously in a background process. This allows a) the SQL Server process to be the main process in the container, which allows graceful shutdown and other goodies, and b) us to only start the SQL Server process once, as opposed to starting, stopping, then starting it again.
+    function initialize_app_database() {
+      # Wait a bit for SQL Server to start. SQL Server's process doesn't provide a clever way to check if it's up or not, and it needs to be up before we can import the application database
+      sleep 30s
+
+      echo "RUNNING BUDIBASE SETUP"
+
+      #run the setup script to create the DB and the schema in the DB
+      /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql
+
+      # Note that the container has been initialized so future starts won't wipe changes to the data
+      touch /tmp/app-initialized
+    }
+    initialize_app_database &
+  fi
+fi
+
+exec "$@"

package/scripts/integrations/mssql/data/setup.sql

@@ -0,0 +1,54 @@
+USE master;
+  
+IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL  
+  DROP TABLE products;  
+GO  
+CREATE TABLE products 
+(  
+  id int IDENTITY(1,1),  
+  name varchar (20),  
+  description varchar(30),
+  CONSTRAINT pk_products PRIMARY KEY NONCLUSTERED (id)
+);
+
+IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL
+  DROP TABLE tasks;
+GO
+CREATE TABLE tasks
+(
+  taskid int IDENTITY(1,1),
+  taskname varchar (20),
+  productid int,
+  CONSTRAINT pk_tasks PRIMARY KEY NONCLUSTERED (taskid),
+  CONSTRAINT fk_products FOREIGN KEY (productid) REFERENCES products (id),
+);
+
+IF OBJECT_ID ('dbo.people', 'U') IS NOT NULL
+  DROP TABLE people;
+GO
+CREATE TABLE people
+(
+  name varchar(30),
+  age varchar(20),
+  CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age)
+);
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Bananas', 'Fruit thing');  
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Meat', 'Animal thing');
+
+INSERT tasks
+  (taskname, productid)
+VALUES
+  ('Processing', 1);
+
+INSERT people
+  (name, age)
+VALUES
+  ('Bob', '30');

package/scripts/integrations/mssql/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3.8"
+services:
+  # password: Passw0rd
+  # user: sa
+  # database: master
+  mssql:
+    image: bb/mssql
+    build:
+      context: .
+      dockerfile: data/Dockerfile
+    ports:
+      - "1433:1433"

package/scripts/integrations/mssql/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

package/scripts/integrations/oracle/docker-compose.yml

@@ -0,0 +1,19 @@
+# For more information see:
+# https://container-registry.oracle.com/
+# - Database > Express
+version: "3.8"
+services:
+  db:
+    restart: always
+    platform: linux/x86_64
+    image: container-registry.oracle.com/database/express:18.4.0-xe
+    environment:
+      ORACLE_PWD: oracle
+    ports:
+      - 1521:1521
+      - 5500:5500
+    volumes:
+      - oracle_data:/opt/oracle/oradata
+
+volumes:
+ oracle_data:

package/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+
+# Must be root to continue
+if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
+
+# Allow for re-runs
+rm -rf /opt/oracle
+
+echo "Installing oracle instant client"
+
+# copy and unzip package
+mkdir -p /opt/oracle
+cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
+cd /opt/oracle
+unzip -qq basiclite-21.4.zip -d .
+rm *.zip
+mv instantclient* instantclient
+
+# update runtime link path
+sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
+ldconfig /etc/ld.so.conf.d
+
+echo "Installation complete"

package/scripts/integrations/oracle/oracle.md

@@ -0,0 +1,90 @@
+# Installation
+
+## Database
+
+**Important**
+- Oracle database is supported only on **x86-64 architecture**
+- Oracle database is **not supported on Mac ARM architecture** (either via docker or linux virtualization)
+
+To install oracle express edition simply run `docker-compose up`
+
+- A single instance pluggable database (PDB) will be created named `xepdb`
+- The default password is configured in the compose file as `oracle`
+  - The `system`, `sys` and `pdbadmin` users all share this password
+
+## Instant Client 
+
+Before oracle can be connected to from nodejs, the oracle client must be installed.
+For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html
+
+**Important**
+- Oracle client is supported only on **x86-64 architecture**
+- Oracle client is **not supported on Mac ARM architecture** 
+
+### Linux
+Run the provided install script for linux from the `server` root path:
+
+```bash
+sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
+```
+For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst
+
+### Mac 
+**This has not yet been tested**
+
+See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst
+
+# Management
+To connect to oracle sql command line:
+
+```bash
+docker exec -it oracle-xe sqlplus -l system/oracle@localhost/xepdb1
+```
+
+To create a new schema (where a user is the same as a schema in oracle) named `sales`:
+
+```sql
+define USERNAME = sales
+
+create user &USERNAME;
+
+alter user &USERNAME
+    default tablespace users
+    temporary tablespace temp
+    quota unlimited on users;
+
+grant create session,
+    create view,
+    create sequence,
+    create procedure,
+    create table,
+    create trigger,
+    create type,
+    create materialized view
+    to &USERNAME;
+```
+
+To set the password for the sales schema use: 
+
+```sql
+define USERNAME = sales
+define PASSWORD = sales
+
+alter user &USERNAME identified by &PASSWORD;
+```
+
+As before the database schema can now be connected to using:
+```bash
+docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1
+```
+
+## HR Schema
+
+The `HR` schema is populated with dummy data by default in oracle for testing purposes. 
+To connect to the HR schema first update the user password and unlock the account by performing
+```sql
+ALTER USER hr ACCOUNT UNLOCK;
+ALTER USER hr IDENTIFIED BY hr
+```
+You should now be able to connect to the hr schema using the credentials hr/hr
+

package/scripts/integrations/postgres/docker-compose.yml

@@ -11,7 +11,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      #- pg_data:/var/lib/postgresql/data/
+      - pg_data:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
@@ -24,5 +24,5 @@ services:
     ports:
       - "5050:80"
 
-#volumes:
-#  pg_data:
+volumes:
+ pg_data:

package/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -15,14 +15,12 @@ const {
   generateAppID,
   getLayoutParams,
   getScreenParams,
-  generateScreenID,
   generateDevAppID,
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
 const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
-const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const {
@@ -44,6 +42,9 @@ const {
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -253,6 +254,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }
@@ -316,8 +318,12 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -328,6 +334,8 @@ exports.sync = async (ctx, next) => {
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
+
+  // replicate prod to dev
   const prodAppId = getDeployedAppID(appId)
 
   try {
@@ -357,6 +365,10 @@ exports.sync = async (ctx, next) => {
   } catch (err) {
     error = err
   }
+
+  // sync the users
+  await syncGlobalUsers(appId)
+
   if (error) {
     ctx.throw(400, error)
   } else {
@@ -380,7 +392,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
@@ -392,9 +407,5 @@ const createEmptyAppPackage = async (ctx, app) => {
     screensAndLayouts.push(await processObject(cloned, app))
   }
 
-  const homeScreen = createHomeScreen(app)
-  homeScreen._id = generateScreenID()
-  screensAndLayouts.push(homeScreen)
-
   await db.bulkDocs(screensAndLayouts)
 }

package/src/api/controllers/auth.js

@@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => {
   }
 
   const user = await getFullUser(ctx, userId)
+  // this shouldn't be returned by the app self
+  delete user.roles
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => {
       // user has a role of some sort, return them
       else if (err.status === 404) {
         const metadata = {
+          ...user,
           _id: userId,
         }
         const dbResp = await db.put(metadata)

package/src/api/controllers/datasource.js

@@ -7,8 +7,9 @@ const {
   BudibaseInternalDB,
   getTableParams,
 } = require("../../db/utils")
+const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
-const { makeExternalQuery } = require("./row/utils")
+const { getDatasourceAndQuery } = require("./row/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -43,13 +44,17 @@ exports.buildSchemaFromDb = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasource = await db.get(ctx.params.datasourceId)
 
-  const tables = await buildSchemaHelper(datasource)
+  const { tables, error } = await buildSchemaHelper(datasource)
   datasource.entities = tables
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
-  ctx.body = datasource
+  const response = { datasource }
+  if (error) {
+    response.error = error
+  }
+  ctx.body = response
 }
 
 exports.update = async function (ctx) {
@@ -71,7 +76,7 @@ exports.update = async function (ctx) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  ctx.body = { datasource }
 }
 
 exports.save = async function (ctx) {
@@ -85,13 +90,15 @@ exports.save = async function (ctx) {
     ...ctx.request.body.datasource,
   }
 
+  let schemaError = null
   if (fetchSchema) {
-    let tables = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildSchemaHelper(datasource)
+    schemaError = error
     datasource.entities = tables
   }
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
@@ -101,17 +108,27 @@ exports.save = async function (ctx) {
     }
   }
 
-  ctx.status = 200
-  ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  const response = { datasource }
+  if (schemaError) {
+    response.error = schemaError
+  }
+  ctx.body = response
 }
 
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   // Delete all queries for the datasource
-  const rows = await db.allDocs(getQueryParams(ctx.params.datasourceId, null))
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  const queries = await db.allDocs(
+    getQueryParams(ctx.params.datasourceId, null)
+  )
+  await db.bulkDocs(
+    queries.rows.map(row => ({
+      _id: row.id,
+      _rev: row.value.rev,
+      _deleted: true,
+    }))
+  )
 
   // delete the datasource
   await db.remove(ctx.params.datasourceId, ctx.params.revId)
@@ -129,12 +146,29 @@ exports.find = async function (ctx) {
 exports.query = async function (ctx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await makeExternalQuery(ctx.appId, queryJson)
+    ctx.body = await getDatasourceAndQuery(ctx.appId, queryJson)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
 
+function getErrorTables(errors, errorType) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error, newError, tables) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
 const buildSchemaHelper = async datasource => {
   const Connector = integrations[datasource.source]
 
@@ -143,5 +177,39 @@ const buildSchemaHelper = async datasource => {
   await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
-  return connector.tables
+  // make sure they all have a display name selected
+  for (let entity of Object.values(datasource.entities)) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
 }

package/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

package/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -24,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {
@@ -106,6 +108,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }