@budibase/server 2.4.44-alpha.1 → 2.4.44-alpha.10

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.4.44-alpha.1",
+  "version": "2.4.44-alpha.10",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -26,6 +26,7 @@
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
     "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
@@ -44,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.4.44-alpha.1",
-    "@budibase/client": "2.4.44-alpha.1",
-    "@budibase/pro": "2.4.44-alpha.0",
-    "@budibase/shared-core": "2.4.44-alpha.1",
-    "@budibase/string-templates": "2.4.44-alpha.1",
-    "@budibase/types": "2.4.44-alpha.1",
+    "@budibase/backend-core": "2.4.44-alpha.10",
+    "@budibase/client": "2.4.44-alpha.10",
+    "@budibase/pro": "2.4.44-alpha.9",
+    "@budibase/shared-core": "2.4.44-alpha.10",
+    "@budibase/string-templates": "2.4.44-alpha.10",
+    "@budibase/types": "2.4.44-alpha.10",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,7 +86,6 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-connect": "2.1.0",
-    "koa-pino-logger": "3.0.0",
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
@@ -99,7 +99,6 @@
     "node-fetch": "2.6.7",
     "open": "8.4.0",
     "pg": "8.5.1",
-    "pino-pretty": "4.0.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "7.2.2",
@@ -176,5 +175,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "2597bbe05cd384874fad4f4da9ffd525117dcacf"
+  "gitHead": "e1b120d9f0399e668c5870e612652a7bc684f47d"
 }

package/scripts/dev/manage.js

@@ -45,6 +45,8 @@ async function init() {
       BB_ADMIN_USER_PASSWORD: "",
       PLUGINS_DIR: "",
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+      HTTP_MIGRATIONS: "0",
+      HTTP_LOGGING: "0",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/src/api/controllers/ops.ts

@@ -0,0 +1,32 @@
+import { Ctx } from "@budibase/types"
+import { logging } from "@budibase/backend-core"
+
+interface LogRequest {
+  message: string
+  data?: any
+}
+
+interface ErrorRequest {
+  message: string
+}
+
+export async function log(ctx: Ctx<LogRequest>) {
+  const body = ctx.request.body
+  console.trace(body.message, body.data)
+  console.debug(body.message, body.data)
+  console.info(body.message, body.data)
+  console.warn(body.message, body.data)
+  console.error(body.message, body.data)
+  ctx.status = 204
+}
+
+export async function alert(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  logging.logAlert(body.message, new Error(body.message))
+  ctx.status = 204
+}
+
+export async function error(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  throw new Error(body.message)
+}

package/src/api/routes/index.ts

@@ -25,6 +25,7 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import opsRoutes from "./ops"
 import Router from "@koa/router"
 import { api as pro } from "@budibase/pro"
 
@@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  opsRoutes,
   scheduleRoutes,
   environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId

package/src/api/routes/ops.ts

@@ -0,0 +1,30 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/ops"
+import { middleware } from "@budibase/backend-core"
+import Joi from "joi"
+
+export function logsValidator() {
+  return middleware.joiValidator.body(
+    Joi.object({
+      message: Joi.string().required(),
+      data: Joi.object(),
+    })
+  )
+}
+
+export function errorValidator() {
+  return middleware.joiValidator.body(
+    Joi.object({
+      message: Joi.string().required(),
+    })
+  )
+}
+
+const router: Router = new Router()
+
+router
+  .post("/api/ops/log", logsValidator(), controller.log)
+  .post("/api/ops/error", errorValidator(), controller.error)
+  .post("/api/ops/alert", errorValidator(), controller.alert)
+
+export default router

package/src/api/routes/tests/user.spec.js

@@ -57,6 +57,7 @@ describe("/users", () => {
     it("should be able to update the user", async () => {
       const user = await config.createUser({ id: `us_update${utils.newid()}` })
       user.roleId = BUILTIN_ROLE_IDS.BASIC
+      delete user._rev
       const res = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
@@ -65,6 +66,46 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.ok).toEqual(true)
     })
+
+    it("should be able to update the user multiple times", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      const res1 = await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      const res = await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      expect(res.body.ok).toEqual(true)
+    })
+
+    it("should require the _rev field for multiple updates", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      await request
+        .put(`/api/users/metadata`)
+        .set(config.defaultHeaders())
+        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
+        .expect(409)
+        .expect("Content-Type", /json/)
+    })
   })
 
   describe("destroy", () => {
@@ -92,6 +133,7 @@ describe("/users", () => {
       expect(res.body.tableId).toBeDefined()
     })
   })
+
   describe("setFlag", () => {
     it("should throw an error if a flag is not provided", async () => {
       await config.createUser()
@@ -101,8 +143,9 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
-
+      expect(res.body.message).toEqual(
+        "Must supply a 'flag' field in request body."
+      )
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -146,8 +189,9 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
-
+      expect(res.body.message).toEqual(
+        "Must supply a 'flag' field in request body."
+      )
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -165,33 +209,37 @@ describe("/users", () => {
   describe("syncUser", () => {
     it("should sync the user", async () => {
       let user = await config.createUser()
-      await config.createApp('New App')
+      await config.createApp("New App")
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
         .expect(200)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual('User synced.')
+      expect(res.body.message).toEqual("User synced.")
     })
 
-
     it("should sync the user when a previous user is specified", async () => {
-      const app1 = await config.createApp('App 1')
-      const app2 = await config.createApp('App 2')
+      const app1 = await config.createApp("App 1")
+      const app2 = await config.createApp("App 2")
 
       let user = await config.createUser({
         builder: false,
         admin: true,
-      roles: { [app1.appId]: 'ADMIN' }
-    })
+        roles: { [app1.appId]: "ADMIN" },
+      })
       let res = await request
         .post(`/api/users/metadata/sync/${user._id}`)
         .set(config.defaultHeaders())
-        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .send({
+          previousUser: {
+            ...user,
+            roles: { ...user.roles, [app2.appId]: "BASIC" },
+          },
+        })
         .expect(200)
         .expect("Content-Type", /json/)
 
-      expect(res.body.message).toEqual('User synced.')
+      expect(res.body.message).toEqual("User synced.")
     })
   })
 })

package/src/app.ts

@@ -2,21 +2,9 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
-if (process.env.ELASTIC_APM_ENABLED) {
-  require("./elasticApm")
-}
-
 // need to load environment first
 import env from "./environment"
 
-// enable APM if configured
-if (process.env.ELASTIC_APM_ENABLED) {
-  const apm = require("elastic-apm-node").start({
-    serviceName: process.env.SERVICE,
-    environment: process.env.BUDIBASE_ENVIRONMENT,
-  })
-}
-
 import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
@@ -53,7 +41,8 @@ app.use(
   })
 )
 
-app.use(middleware.logging)
+app.use(middleware.correlation)
+app.use(middleware.pino)
 app.use(userAgent)
 
 if (env.isProd()) {

package/src/environment.ts

@@ -62,7 +62,6 @@ const environment = {
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
-  LOG_LEVEL: process.env.LOG_LEVEL,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
     parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,

package/src/integrations/rest.ts

@@ -151,6 +151,9 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
+      } else if (contentType.includes("application/pdf")) {
+        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
+        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -9,6 +9,6 @@ export const run = async () => {
 
   // sync app count
   const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,8 +18,7 @@ export const run = async () => {
   })
 
   // sync row count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  console.log(`Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/startup.ts

@@ -16,13 +16,10 @@ import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
-const pino = require("koa-pino-logger")
 
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(logging.pinoSettings()))
-
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -48,8 +45,10 @@ async function initPro() {
 }
 
 function shutdown(server?: any) {
-  server.close()
-  server.destroy()
+  if (server) {
+    server.close()
+    server.destroy()
+  }
 }
 
 export async function startup(app?: any, server?: any) {
@@ -72,11 +71,39 @@ export async function startup(app?: any, server?: any) {
       await migrations.migrate()
     } catch (e) {
       logging.logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
+      shutdown(server)
     }
   }
 
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+
   // check and create admin user if required
+  // this must be run after the api has been initialised due to
+  // the app user sync
   if (
     env.SELF_HOSTED &&
     !env.MULTI_TENANCY &&
@@ -103,34 +130,8 @@ export async function startup(app?: any, server?: any) {
         )
       } catch (e) {
         logging.logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
+        shutdown(server)
       }
     }
   }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // get the references to the queue promises, don't await as
-  // they will never end, unless the processing stops
-  let queuePromises = []
-  // configure events to use the pro audit log write
-  // can't integrate directly into backend-core due to cyclic issues
-  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
-  queuePromises.push(automations.init())
-  queuePromises.push(initPro())
-  if (app) {
-    // bring routes online as final step once everything ready
-    await initRoutes(app)
-  }
 }

package/src/tests/jestEnv.ts

@@ -6,7 +6,6 @@ process.env.MULTI_TENANCY = "1"
 // @ts-ignore
 process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"

package/src/tests/jestSetup.ts

@@ -1,4 +1,3 @@
-import "./logging"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"

package/src/threads/automation.ts

@@ -34,8 +34,8 @@ const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   const binding = automationUtils.typecastForLooping(loopStep, input)
-  if (!loopStep || !binding) {
-    return 1
+  if (!binding) {
+    return 0
   }
   if (Array.isArray(binding)) {
     return binding.length
@@ -43,7 +43,7 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   if (typeof binding === "string") {
     return automationUtils.stringSplit(binding).length
   }
-  return 1
+  return 0
 }
 
 /**
@@ -423,13 +423,25 @@ class Orchestrator {
         }
       }
 
+      if (loopStep && iterations === 0) {
+        loopStep = undefined
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: { status: AutomationStatus.NO_ITERATIONS, success: true },
+          inputs: {},
+        })
+
+        this._context.steps.splice(loopStepNumber, 1)
+        iterations = 1
+      }
+
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
       if (wasLoopStep && !loopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
-
       if (loopSteps && loopSteps.length) {
         let tempOutput = {
           success: true,

package/dist/elasticApm.js

@@ -1,14 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const elastic_apm_node_1 = __importDefault(require("elastic-apm-node"));
-// enable APM if configured
-if (process.env.ELASTIC_APM_ENABLED) {
-    console.log("Starting elastic-apm-node");
-    elastic_apm_node_1.default.start({
-        serviceName: process.env.SERVICE,
-        environment: process.env.BUDIBASE_ENVIRONMENT,
-    });
-}

package/scripts/likeCypress.ts

@@ -1,35 +0,0 @@
-/******************************************************
- * This script just makes it easy to re-create        *
- * a cypress like environment for testing the backend *
- ******************************************************/
-import path from "path"
-const tmpdir = path.join(require("os").tmpdir(), ".budibase")
-
-const SERVER_PORT = "4100"
-const WORKER_PORT = "4200"
-
-// @ts-ignore
-process.env.NODE_ENV = "cypress"
-process.env.ENABLE_ANALYTICS = "0"
-process.env.JWT_SECRET = "budibase"
-process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
-process.env.SELF_HOSTED = "1"
-process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
-process.env.MINIO_URL = `http://localhost:4004`
-process.env.MINIO_ACCESS_KEY = "budibase"
-process.env.MINIO_SECRET_KEY = "budibase"
-process.env.COUCH_DB_USER = "budibase"
-process.env.COUCH_DB_PASSWORD = "budibase"
-process.env.INTERNAL_API_KEY = "budibase"
-process.env.ALLOW_DEV_AUTOMATIONS = "1"
-
-// don't make this a variable or top level require
-// it will cause environment module to be loaded prematurely
-
-// override the port with the worker port temporarily
-process.env.PORT = WORKER_PORT
-const worker = require("../../worker/src/index")
-
-// override the port with the server port
-process.env.PORT = SERVER_PORT
-const server = require("../src/app")

package/src/elasticApm.ts

@@ -1,10 +0,0 @@
-import apm from "elastic-apm-node"
-
-// enable APM if configured
-if (process.env.ELASTIC_APM_ENABLED) {
-  console.log("Starting elastic-apm-node")
-  apm.start({
-    serviceName: process.env.SERVICE,
-    environment: process.env.BUDIBASE_ENVIRONMENT,
-  })
-}

package/src/tests/logging.ts

@@ -1,34 +0,0 @@
-export enum LogLevel {
-  TRACE = "trace",
-  DEBUG = "debug",
-  INFO = "info",
-  WARN = "warn",
-  ERROR = "error",
-}
-
-const LOG_INDEX: { [key in LogLevel]: number } = {
-  [LogLevel.TRACE]: 1,
-  [LogLevel.DEBUG]: 2,
-  [LogLevel.INFO]: 3,
-  [LogLevel.WARN]: 4,
-  [LogLevel.ERROR]: 5,
-}
-
-const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
-
-if (setIndex > LOG_INDEX.trace) {
-  global.console.trace = jest.fn()
-}
-
-if (setIndex > LOG_INDEX.debug) {
-  global.console.debug = jest.fn()
-}
-
-if (setIndex > LOG_INDEX.info) {
-  global.console.info = jest.fn()
-  global.console.log = jest.fn()
-}
-
-if (setIndex > LOG_INDEX.warn) {
-  global.console.warn = jest.fn()
-}