@budibase/server 2.5.5-alpha.4 → 2.5.5

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.5-alpha.4",
+  "version": "2.5.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -26,7 +26,6 @@
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
     "dev:builder": "yarn run dev:stack:up && nodemon",
-    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
@@ -45,12 +44,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.5.5-alpha.4",
-    "@budibase/client": "2.5.5-alpha.4",
-    "@budibase/pro": "2.5.5-alpha.2",
-    "@budibase/shared-core": "2.5.5-alpha.4",
-    "@budibase/string-templates": "2.5.5-alpha.4",
-    "@budibase/types": "2.5.5-alpha.4",
+    "@budibase/backend-core": "^2.5.5",
+    "@budibase/client": "^2.5.5",
+    "@budibase/pro": "2.5.4",
+    "@budibase/shared-core": "^2.5.5",
+    "@budibase/string-templates": "^2.5.5",
+    "@budibase/types": "^2.5.5",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -86,6 +85,7 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-connect": "2.1.0",
+    "koa-pino-logger": "3.0.0",
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
@@ -99,6 +99,7 @@
     "node-fetch": "2.6.7",
     "open": "8.4.0",
     "pg": "8.5.1",
+    "pino-pretty": "4.0.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "7.2.2",
@@ -115,10 +116,11 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "vm2": "3.9.16",
+    "vm2": "^3.9.15",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
-    "yargs": "13.2.4"
+    "xml2js": "0.4.23",
+    "yargs": "13.2.4",
+    "zlib": "1.0.5"
   },
   "devDependencies": {
     "@babel/core": "7.17.4",
@@ -147,7 +149,7 @@
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
-    "babel-jest": "29.5.0",
+    "babel-jest": "27.5.1",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
@@ -155,7 +157,6 @@
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
-    "jest-runner": "29.5.0",
     "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
@@ -175,5 +176,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "ee3c525f17f68904ac7a992d245f6ef57568146b"
+  "gitHead": "ed22164a05a38f41d3327b2ad18f23ddf857a062"
 }

package/scripts/dev/manage.js

@@ -45,8 +45,6 @@ async function init() {
       BB_ADMIN_USER_PASSWORD: "",
       PLUGINS_DIR: "",
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      HTTP_MIGRATIONS: "0",
-      HTTP_LOGGING: "0",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/scripts/integrations/mssql/data/entrypoint.sh

@@ -11,7 +11,6 @@ if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then
 
       echo "RUNNING BUDIBASE SETUP"
 
-      cat setup.sql
       #run the setup script to create the DB and the schema in the DB
       /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql
 

package/scripts/integrations/mssql/data/setup.sql

@@ -34,7 +34,7 @@ GO
 CREATE TABLE people
 (
   name varchar(30) NOT NULL,
-  age int default 20 NOT NULL,
+  age varchar(20),
   CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age)
 );
   
@@ -50,22 +50,22 @@ VALUES
   ('Processing', 1);
 
 INSERT INTO people (name, age)
-VALUES ('Bob', 30),
-  ('Bert', 10),
-  ('Jack', 12),
-  ('Mike', 31),
-  ('Dave', 44),
-  ('Jim', 43),
-  ('Kerry', 32),
-  ('Julie', 12),
-  ('Kim', 55),
-  ('Andy', 33),
-  ('John', 22),
-  ('Ruth', 66),
-  ('Robert', 88),
-  ('Bobert', 99),
-  ('Jan', 22),
-  ('Megan', 11);
+VALUES ('Bob', '30'),
+  ('Bert', '10'),
+  ('Jack', '12'),
+  ('Mike', '31'),
+  ('Dave', '44'),
+  ('Jim', '43'),
+  ('Kerry', '32'),
+  ('Julie', '12'),
+  ('Kim', '55'),
+  ('Andy', '33'),
+  ('John', '22'),
+  ('Ruth', '66'),
+  ('Robert', '88'),
+  ('Bobert', '99'),
+  ('Jan', '22'),
+  ('Megan', '11');
 
 
 IF OBJECT_ID ('Chains.sizes', 'U') IS NOT NULL

package/scripts/integrations/mysql/init.sql

@@ -3,7 +3,7 @@ USE main;
 CREATE TABLE Persons (
     PersonID int NOT NULL AUTO_INCREMENT,
     CreatedAt datetime,
-    Age float DEFAULT 20 NOT NULL,
+    Age float,
     LastName varchar(255),
     FirstName varchar(255),
     Address varchar(255),

package/scripts/integrations/postgres/init.sql

@@ -8,7 +8,6 @@ CREATE TABLE Persons (
     FirstName varchar(255),
     Address varchar(255),
     City varchar(255) DEFAULT 'Belfast',
-    Age INTEGER DEFAULT 20 NOT NULL,
     Type person_job
 );
 CREATE TABLE Tasks (

package/scripts/likeCypress.ts

@@ -0,0 +1,35 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+import path from "path"
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const SERVER_PORT = "4100"
+const WORKER_PORT = "4200"
+
+// @ts-ignore
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "0"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:4004`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+
+// override the port with the worker port temporarily
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+
+// override the port with the server port
+process.env.PORT = SERVER_PORT
+const server = require("../src/app")

package/src/api/routes/index.ts

@@ -25,7 +25,6 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
-import opsRoutes from "./ops"
 import Router from "@koa/router"
 import { api as pro } from "@budibase/pro"
 
@@ -64,7 +63,6 @@ export const mainRoutes: Router[] = [
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
-  opsRoutes,
   scheduleRoutes,
   environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId

package/src/api/routes/tests/automation.spec.js

@@ -19,14 +19,11 @@ describe("/automations", () => {
 
   afterAll(setup.afterAll)
 
-  beforeAll(async () => {
+  // For some reason this cannot be a beforeAll or the test "tests the automation successfully" fail
+  beforeEach(async () => {
     await config.init()
   })
 
-  beforeEach(() => {
-    events.automation.deleted.mockClear()
-  })
-
   describe("get definitions", () => {
     it("returns a list of definitions for actions", async () => {
       const res = await request

package/src/api/routes/tests/user.spec.js

@@ -57,7 +57,6 @@ describe("/users", () => {
     it("should be able to update the user", async () => {
       const user = await config.createUser({ id: `us_update${utils.newid()}` })
       user.roleId = BUILTIN_ROLE_IDS.BASIC
-      delete user._rev
       const res = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
@@ -66,46 +65,6 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.ok).toEqual(true)
     })
-
-    it("should be able to update the user multiple times", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      const res1 = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      expect(res.body.ok).toEqual(true)
-    })
-
-    it("should require the _rev field for multiple updates", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(409)
-        .expect("Content-Type", /json/)
-    })
   })
 
   describe("destroy", () => {
@@ -133,7 +92,6 @@ describe("/users", () => {
       expect(res.body.tableId).toBeDefined()
     })
   })
-
   describe("setFlag", () => {
     it("should throw an error if a flag is not provided", async () => {
       await config.createUser()
@@ -143,9 +101,8 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
+      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
+
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -189,9 +146,8 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
+      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
+
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -209,37 +165,33 @@ describe("/users", () => {
   describe("syncUser", () => {
     it("should sync the user", async () => {
       let user = await config.createUser()
-      await config.createApp("New App")
+      await config.createApp('New App')
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
         .expect(200)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("User synced.")
+      expect(res.body.message).toEqual('User synced.')
     })
 
+
     it("should sync the user when a previous user is specified", async () => {
-      const app1 = await config.createApp("App 1")
-      const app2 = await config.createApp("App 2")
+      const app1 = await config.createApp('App 1')
+      const app2 = await config.createApp('App 2')
 
       let user = await config.createUser({
         builder: false,
         admin: true,
-        roles: { [app1.appId]: "ADMIN" },
-      })
+      roles: { [app1.appId]: 'ADMIN' }
+    })
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
-        .send({
-          previousUser: {
-            ...user,
-            roles: { ...user.roles, [app2.appId]: "BASIC" },
-          },
-        })
+        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
         .expect(200)
         .expect("Content-Type", /json/)
 
-      expect(res.body.message).toEqual("User synced.")
+      expect(res.body.message).toEqual('User synced.')
     })
   })
 })

package/src/app.ts

@@ -2,9 +2,21 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
+if (process.env.ELASTIC_APM_ENABLED) {
+  require("./elasticApm")
+}
+
 // need to load environment first
 import env from "./environment"
 
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  const apm = require("elastic-apm-node").start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}
+
 import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
@@ -41,8 +53,7 @@ app.use(
   })
 )
 
-app.use(middleware.correlation)
-app.use(middleware.pino)
+app.use(middleware.logging)
 app.use(userAgent)
 
 if (env.isProd()) {

package/src/elasticApm.ts

@@ -0,0 +1,10 @@
+import apm from "elastic-apm-node"
+
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  console.log("Starting elastic-apm-node")
+  apm.start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}

package/src/environment.ts

@@ -62,6 +62,7 @@ const environment = {
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
+  LOG_LEVEL: process.env.LOG_LEVEL,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
     parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,

package/src/integrations/microsoftSqlServer.ts

@@ -243,14 +243,11 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         if (typeof name !== "string") {
           continue
         }
-        const hasDefault = def.COLUMN_DEFAULT
-        const isAuto = !!autoColumns.find(col => col === name)
-        const required = !!requiredColumns.find(col => col === name)
         schema[name] = {
-          autocolumn: isAuto,
+          autocolumn: !!autoColumns.find(col => col === name),
           name: name,
           constraints: {
-            presence: required && !isAuto && !hasDefault,
+            presence: requiredColumns.find(col => col === name),
           },
           ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,

package/src/integrations/mysql.ts

@@ -229,15 +229,13 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
           if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
             primaryKeys.push(columnName)
           }
-          const hasDefault = column.Default != null
+          const constraints = {
+            presence: column.Null !== "YES",
+          }
           const isAuto: boolean =
             typeof column.Extra === "string" &&
             (column.Extra === "auto_increment" ||
               column.Extra.toLowerCase().includes("generated"))
-          const required = column.Null !== "YES"
-          const constraints = {
-            presence: required && !isAuto && !hasDefault,
-          }
           schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,

package/src/integrations/postgres.ts

@@ -262,17 +262,15 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.identity_start ||
           column.identity_increment
         )
-        const hasDefault = column.column_default != null
-        const hasNextVal =
+        const constraints = {
+          presence: column.is_nullable === "NO",
+        }
+        const hasDefault =
           typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
         const isGenerated =
           column.is_generated && column.is_generated !== "NEVER"
-        const isAuto: boolean = hasNextVal || identity || isGenerated
-        const required = column.is_nullable === "NO"
-        const constraints = {
-          presence: required && !hasDefault && !isGenerated,
-        }
+        const isAuto: boolean = hasDefault || identity || isGenerated
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,

package/src/integrations/redis.ts

@@ -6,7 +6,6 @@ interface RedisConfig {
   port: number
   username: string
   password?: string
-  db?: number
 }
 
 const SCHEMA: Integration = {
@@ -33,12 +32,6 @@ const SCHEMA: Integration = {
       type: "password",
       required: false,
     },
-    db: {
-      type: "number",
-      required: false,
-      display: "DB",
-      default: 0,
-    },
   },
   query: {
     create: {
@@ -95,7 +88,6 @@ class RedisIntegration {
       port: this.config.port,
       username: this.config.username,
       password: this.config.password,
-      db: this.config.db,
     })
   }
 

package/src/integrations/rest.ts

@@ -151,9 +151,6 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
-      } else if (contentType.includes("application/pdf")) {
-        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
-        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -9,6 +9,6 @@ export const run = async () => {
 
   // sync app count
   const tenantId = tenancy.getTenantId()
-  console.log(`Syncing app count: ${appCount}`)
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { db as dbCore } from "@budibase/backend-core"
+import { tenancy, db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,7 +18,8 @@ export const run = async () => {
   })
 
   // sync row count
-  console.log(`Syncing row count: ${rowCount}`)
+  const tenantId = tenancy.getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -24,7 +24,7 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await context.doInAppContext(app1!.appId, async () => {
+      await context.doInAppContext(app1.appId, async () => {
         await config.createTable()
         await config.createRow()
       })
@@ -43,7 +43,7 @@ describe("syncRows", () => {
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
       expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
+        usageDoc.apps?.[dbCore.getProdAppID(app1.appId)].usageQuota.rows
       ).toEqual(1)
       expect(
         usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows