@budibase/server 2.5.3 → 2.5.5-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.3",
+  "version": "2.5.5-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -26,6 +26,7 @@
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
     "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
@@ -44,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.5.3",
-    "@budibase/client": "^2.5.3",
-    "@budibase/pro": "2.5.2",
-    "@budibase/shared-core": "^2.5.3",
-    "@budibase/string-templates": "^2.5.3",
-    "@budibase/types": "^2.5.3",
+    "@budibase/backend-core": "2.5.5-alpha.0",
+    "@budibase/client": "2.5.5-alpha.0",
+    "@budibase/pro": "2.5.4",
+    "@budibase/shared-core": "2.5.5-alpha.0",
+    "@budibase/string-templates": "2.5.5-alpha.0",
+    "@budibase/types": "2.5.5-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,7 +86,6 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-connect": "2.1.0",
-    "koa-pino-logger": "3.0.0",
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
@@ -99,7 +99,6 @@
     "node-fetch": "2.6.7",
     "open": "8.4.0",
     "pg": "8.5.1",
-    "pino-pretty": "4.0.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "7.2.2",
@@ -116,9 +115,9 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "vm2": "^3.9.15",
+    "vm2": "3.9.16",
     "worker-farm": "1.7.0",
-    "xml2js": "0.4.23",
+    "xml2js": "0.5.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -149,7 +148,7 @@
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
-    "babel-jest": "27.5.1",
+    "babel-jest": "29.5.0",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
@@ -157,6 +156,7 @@
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
+    "jest-runner": "29.5.0",
     "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
@@ -176,5 +176,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "468a300cda385b3fe87ed7f1f51749282015612a"
+  "gitHead": "84e8cc00315e223433e388b72fae3bd048ff5760"
 }

package/scripts/dev/manage.js

@@ -45,6 +45,8 @@ async function init() {
       BB_ADMIN_USER_PASSWORD: "",
       PLUGINS_DIR: "",
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+      HTTP_MIGRATIONS: "0",
+      HTTP_LOGGING: "0",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/src/api/controllers/ops.ts

@@ -0,0 +1,32 @@
+import { Ctx } from "@budibase/types"
+import { logging } from "@budibase/backend-core"
+
+interface LogRequest {
+  message: string
+  data?: any
+}
+
+interface ErrorRequest {
+  message: string
+}
+
+export async function log(ctx: Ctx<LogRequest>) {
+  const body = ctx.request.body
+  console.trace(body.message, body.data)
+  console.debug(body.message, body.data)
+  console.info(body.message, body.data)
+  console.warn(body.message, body.data)
+  console.error(body.message, body.data)
+  ctx.status = 204
+}
+
+export async function alert(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  logging.logAlert(body.message, new Error(body.message))
+  ctx.status = 204
+}
+
+export async function error(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  throw new Error(body.message)
+}

package/src/api/routes/index.ts

@@ -25,6 +25,7 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import opsRoutes from "./ops"
 import Router from "@koa/router"
 import { api as pro } from "@budibase/pro"
 
@@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  opsRoutes,
   scheduleRoutes,
   environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId

package/src/api/routes/ops.ts

@@ -0,0 +1,30 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/ops"
+import { middleware } from "@budibase/backend-core"
+import Joi from "joi"
+
+export function logsValidator() {
+  return middleware.joiValidator.body(
+    Joi.object({
+      message: Joi.string().required(),
+      data: Joi.object(),
+    })
+  )
+}
+
+export function errorValidator() {
+  return middleware.joiValidator.body(
+    Joi.object({
+      message: Joi.string().required(),
+    })
+  )
+}
+
+const router: Router = new Router()
+
+router
+  .post("/api/ops/log", logsValidator(), controller.log)
+  .post("/api/ops/error", errorValidator(), controller.error)
+  .post("/api/ops/alert", errorValidator(), controller.alert)
+
+export default router

package/src/api/routes/tests/automation.spec.js

@@ -19,11 +19,14 @@ describe("/automations", () => {
 
   afterAll(setup.afterAll)
 
-  // For some reason this cannot be a beforeAll or the test "tests the automation successfully" fail
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 
+  beforeEach(() => {
+    events.automation.deleted.mockClear()
+  })
+
   describe("get definitions", () => {
     it("returns a list of definitions for actions", async () => {
       const res = await request

package/src/api/routes/tests/user.spec.js

@@ -57,6 +57,7 @@ describe("/users", () => {
     it("should be able to update the user", async () => {
       const user = await config.createUser({ id: `us_update${utils.newid()}` })
       user.roleId = BUILTIN_ROLE_IDS.BASIC
+      delete user._rev
       const res = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
@@ -65,6 +66,46 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.ok).toEqual(true)
     })
+
+    it("should be able to update the user multiple times", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      const res1 = await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      const res = await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      expect(res.body.ok).toEqual(true)
+    })
+
+    it("should require the _rev field for multiple updates", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
+        .expect(409)
+        .expect("Content-Type", /json/)
+    })
   })
 
   describe("destroy", () => {
@@ -92,6 +133,7 @@ describe("/users", () => {
       expect(res.body.tableId).toBeDefined()
     })
   })
+
   describe("setFlag", () => {
     it("should throw an error if a flag is not provided", async () => {
       await config.createUser()
@@ -101,8 +143,9 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
-
+      expect(res.body.message).toEqual(
+        "Must supply a 'flag' field in request body."
+      )
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -146,8 +189,9 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
-
+      expect(res.body.message).toEqual(
+        "Must supply a 'flag' field in request body."
+      )
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -165,33 +209,37 @@ describe("/users", () => {
   describe("syncUser", () => {
     it("should sync the user", async () => {
       let user = await config.createUser()
-      await config.createApp('New App')
+      await config.createApp("New App")
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
         .expect(200)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual('User synced.')
+      expect(res.body.message).toEqual("User synced.")
     })
 
-
     it("should sync the user when a previous user is specified", async () => {
-      const app1 = await config.createApp('App 1')
-      const app2 = await config.createApp('App 2')
+      const app1 = await config.createApp("App 1")
+      const app2 = await config.createApp("App 2")
 
       let user = await config.createUser({
         builder: false,
         admin: true,
-      roles: { [app1.appId]: 'ADMIN' }
-    })
+        roles: { [app1.appId]: "ADMIN" },
+      })
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
-        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .send({
+          previousUser: {
+            ...user,
+            roles: { ...user.roles, [app2.appId]: "BASIC" },
+          },
+        })
         .expect(200)
         .expect("Content-Type", /json/)
 
-      expect(res.body.message).toEqual('User synced.')
+      expect(res.body.message).toEqual("User synced.")
     })
   })
 })

package/src/app.ts

@@ -2,21 +2,9 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
-if (process.env.ELASTIC_APM_ENABLED) {
-  require("./elasticApm")
-}
-
 // need to load environment first
 import env from "./environment"
 
-// enable APM if configured
-if (process.env.ELASTIC_APM_ENABLED) {
-  const apm = require("elastic-apm-node").start({
-    serviceName: process.env.SERVICE,
-    environment: process.env.BUDIBASE_ENVIRONMENT,
-  })
-}
-
 import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
@@ -53,7 +41,8 @@ app.use(
   })
 )
 
-app.use(middleware.logging)
+app.use(middleware.correlation)
+app.use(middleware.pino)
 app.use(userAgent)
 
 if (env.isProd()) {

package/src/environment.ts

@@ -62,7 +62,6 @@ const environment = {
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
-  LOG_LEVEL: process.env.LOG_LEVEL,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
     parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,

package/src/integrations/redis.ts

@@ -6,6 +6,7 @@ interface RedisConfig {
   port: number
   username: string
   password?: string
+  db?: number
 }
 
 const SCHEMA: Integration = {
@@ -32,6 +33,12 @@ const SCHEMA: Integration = {
       type: "password",
       required: false,
     },
+    db: {
+      type: "number",
+      required: false,
+      display: "DB",
+      default: 0,
+    },
   },
   query: {
     create: {
@@ -88,6 +95,7 @@ class RedisIntegration {
       port: this.config.port,
       username: this.config.username,
       password: this.config.password,
+      db: this.config.db,
     })
   }
 

package/src/integrations/rest.ts

@@ -151,6 +151,9 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
+      } else if (contentType.includes("application/pdf")) {
+        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
+        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -9,6 +9,6 @@ export const run = async () => {
 
   // sync app count
   const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,8 +18,7 @@ export const run = async () => {
   })
 
   // sync row count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  console.log(`Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -24,7 +24,7 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await context.doInAppContext(app1.appId, async () => {
+      await context.doInAppContext(app1!.appId, async () => {
         await config.createTable()
         await config.createRow()
       })
@@ -43,7 +43,7 @@ describe("syncRows", () => {
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
       expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1.appId)].usageQuota.rows
+        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
       ).toEqual(1)
       expect(
         usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows

package/src/startup.ts

@@ -16,13 +16,10 @@ import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
-const pino = require("koa-pino-logger")
 
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(logging.pinoSettings()))
-
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -48,8 +45,10 @@ async function initPro() {
 }
 
 function shutdown(server?: any) {
-  server.close()
-  server.destroy()
+  if (server) {
+    server.close()
+    server.destroy()
+  }
 }
 
 export async function startup(app?: any, server?: any) {
@@ -72,11 +71,39 @@ export async function startup(app?: any, server?: any) {
       await migrations.migrate()
     } catch (e) {
       logging.logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
+      shutdown(server)
     }
   }
 
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+
   // check and create admin user if required
+  // this must be run after the api has been initialised due to
+  // the app user sync
   if (
     env.SELF_HOSTED &&
     !env.MULTI_TENANCY &&
@@ -103,34 +130,8 @@ export async function startup(app?: any, server?: any) {
         )
       } catch (e) {
         logging.logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
+        shutdown(server)
       }
     }
   }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // get the references to the queue promises, don't await as
-  // they will never end, unless the processing stops
-  let queuePromises = []
-  // configure events to use the pro audit log write
-  // can't integrate directly into backend-core due to cyclic issues
-  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
-  queuePromises.push(automations.init())
-  queuePromises.push(initPro())
-  if (app) {
-    // bring routes online as final step once everything ready
-    await initRoutes(app)
-  }
 }

package/src/tests/jestEnv.ts

@@ -6,7 +6,6 @@ process.env.MULTI_TENANCY = "1"
 // @ts-ignore
 process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"

package/src/tests/jestSetup.ts

@@ -1,4 +1,3 @@
-import "./logging"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"

package/src/threads/automation.ts

@@ -34,8 +34,8 @@ const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   const binding = automationUtils.typecastForLooping(loopStep, input)
-  if (!loopStep || !binding) {
-    return 1
+  if (!binding) {
+    return 0
   }
   if (Array.isArray(binding)) {
     return binding.length
@@ -43,7 +43,7 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   if (typeof binding === "string") {
     return automationUtils.stringSplit(binding).length
   }
-  return 1
+  return 0
 }
 
 /**
@@ -423,13 +423,25 @@ class Orchestrator {
         }
       }
 
+      if (loopStep && iterations === 0) {
+        loopStep = undefined
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: { status: AutomationStatus.NO_ITERATIONS, success: true },
+          inputs: {},
+        })
+
+        this._context.steps.splice(loopStepNumber, 1)
+        iterations = 1
+      }
+
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
       if (wasLoopStep && !loopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
-
       if (loopSteps && loopSteps.length) {
         let tempOutput = {
           success: true,