@budibase/backend-core 2.5.6-alpha.6 → 2.5.6

package/src/middleware/authenticated.ts

@@ -96,15 +96,9 @@ export default function (
     }
     try {
       // check the actual user is authenticated first, try header or cookie
-      let headerToken = ctx.request.headers[Header.TOKEN]
-
+      const headerToken = ctx.request.headers[Header.TOKEN]
       const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
-      let apiKey = ctx.request.headers[Header.API_KEY]
-
-      if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {
-        apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1]
-      }
-
+      const apiKey = ctx.request.headers[Header.API_KEY]
       const tenantId = ctx.request.headers[Header.TENANT_ID]
       let authenticated = false,
         user = null,

package/src/middleware/errorHandling.ts

@@ -1,5 +1,6 @@
 import { APIError } from "@budibase/types"
 import * as errors from "../errors"
+import env from "../environment"
 
 export async function errorHandling(ctx: any, next: any) {
   try {
@@ -8,10 +9,9 @@ export async function errorHandling(ctx: any, next: any) {
     const status = err.status || err.statusCode || 500
     ctx.status = status
 
-    if (status >= 400 && status < 500) {
-      console.warn(err)
-    } else {
-      console.error(err)
+    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
+      ctx.log.error(err)
+      console.trace(err)
     }
 
     const error = errors.getPublicError(err)

package/src/middleware/index.ts

@@ -14,8 +14,7 @@ export { default as csrf } from "./csrf"
 export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
-export { default as pino } from "../logging/pino/middleware"
-export { default as correlation } from "../logging/correlation/middleware"
+export { default as logging } from "./logging"
 export { default as errorHandling } from "./errorHandling"
 export { default as querystringToBody } from "./querystringToBody"
 export * as joiValidator from "./joi-validator"

package/src/middleware/logging.ts

@@ -0,0 +1,90 @@
+const correlator = require("correlation-id")
+import { Header } from "../constants"
+import { v4 as uuid } from "uuid"
+import * as context from "../context"
+
+const debug = console.warn
+const trace = console.trace
+const log = console.log
+const info = console.info
+const warn = console.warn
+const error = console.error
+
+const getTenantId = () => {
+  let tenantId
+  try {
+    tenantId = context.getTenantId()
+  } catch (e: any) {
+    // do nothing
+  }
+  return tenantId
+}
+
+const getAppId = () => {
+  let appId
+  try {
+    appId = context.getAppId()
+  } catch (e) {
+    // do nothing
+  }
+  return appId
+}
+
+const getIdentityId = () => {
+  let identityId
+  try {
+    const identity = context.getIdentity()
+    identityId = identity?._id
+  } catch (e) {
+    // do nothing
+  }
+  return identityId
+}
+
+const print = (fn: any, data: any[]) => {
+  let message = ""
+
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    message = message + `[correlationId=${correlator.getId()}]`
+  }
+
+  const tenantId = getTenantId()
+  if (tenantId) {
+    message = message + ` [tenantId=${tenantId}]`
+  }
+
+  const appId = getAppId()
+  if (appId) {
+    message = message + ` [appId=${appId}]`
+  }
+
+  const identityId = getIdentityId()
+  if (identityId) {
+    message = message + ` [identityId=${identityId}]`
+  }
+
+  if (!process.env.CI) {
+    fn(message, data)
+  }
+}
+
+const logging = (ctx: any, next: any) => {
+  // use the provided correlation id header if present
+  let correlationId = ctx.headers[Header.CORRELATION_ID]
+  if (!correlationId) {
+    correlationId = uuid()
+  }
+
+  return correlator.withId(correlationId, () => {
+    console.debug = data => print(debug, data)
+    console.trace = data => print(trace, data)
+    console.log = data => print(log, data)
+    console.info = data => print(info, data)
+    console.warn = data => print(warn, data)
+    console.error = data => print(error, data)
+    return next()
+  })
+}
+
+export default logging

package/src/middleware/passport/sso/oidc.ts

@@ -1,7 +1,6 @@
 import fetch from "node-fetch"
 import * as sso from "./sso"
 import { ssoCallbackUrl } from "../utils"
-import { validEmail } from "../../../utils"
 import {
   ConfigType,
   OIDCInnerConfig,
@@ -12,7 +11,6 @@ import {
   JwtClaims,
   SaveSSOUserFunction,
 } from "@budibase/types"
-
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
 export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
@@ -88,6 +86,15 @@ function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) {
   )
 }
 
+function validEmail(value: string) {
+  return (
+    value &&
+    !!value.match(
+      /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
+    )
+  )
+}
+
 /**
  * Create an instance of the oidc passport strategy. This wrapper fetches the configuration
  * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.

package/src/middleware/passport/sso/tests/sso.spec.ts

@@ -1,5 +1,4 @@
-import { structures, mocks } from "../../../../../tests"
-import { testEnv } from "../../../../../tests/extra"
+import { structures, testEnv, mocks } from "../../../../../tests"
 import { SSOAuthDetails, User } from "@budibase/types"
 
 import { HTTPError } from "../../../../errors"

package/src/migrations/migrations.ts

@@ -99,7 +99,9 @@ export const runMigration = async (
           options.force[migrationType] &&
           options.force[migrationType].includes(migrationName)
         ) {
-          log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+          )
         } else {
           // no force, exit
           return
@@ -109,7 +111,7 @@ export const runMigration = async (
       // check if the migration is not a no-op
       if (!options.noOp) {
         log(
-          `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
         )
 
         if (migration.preventRetry) {
@@ -129,7 +131,9 @@ export const runMigration = async (
           await migration.fn(db)
         }
 
-        log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+        )
       }
 
       // mark as complete
@@ -137,7 +141,7 @@ export const runMigration = async (
       await db.put(doc)
     } catch (err) {
       console.error(
-        `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
         err
       )
       throw err

package/src/migrations/tests/__snapshots__/migrations.spec.ts.snap

@@ -1,7 +1,7 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`migrations should match snapshot 1`] = `
-{
+Object {
   "_id": "migrations",
   "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
   "createdAt": "2020-01-01T00:00:00.000Z",

package/src/migrations/tests/migrations.spec.ts

@@ -1,4 +1,4 @@
-import { testEnv, DBTestConfiguration } from "../../../tests/extra"
+import { testEnv, DBTestConfiguration } from "../../../tests"
 import * as migrations from "../index"
 import * as context from "../../context"
 import { MigrationType } from "@budibase/types"

package/src/objectStore/buckets/tests/app.spec.ts

@@ -1,6 +1,6 @@
 import * as app from "../app"
 import { getAppFileUrl } from "../app"
-import { testEnv } from "../../../../tests/extra"
+import { testEnv } from "../../../../tests"
 
 describe("app", () => {
   beforeEach(() => {

package/src/objectStore/buckets/tests/global.spec.ts

@@ -1,5 +1,5 @@
 import * as global from "../global"
-import { testEnv } from "../../../../tests/extra"
+import { testEnv } from "../../../../tests"
 
 describe("global", () => {
   describe("getGlobalFileUrl", () => {

package/src/objectStore/buckets/tests/plugins.spec.ts

@@ -1,6 +1,5 @@
 import * as plugins from "../plugins"
-import { structures } from "../../../../tests"
-import { testEnv } from "../../../../tests/extra"
+import { structures, testEnv } from "../../../../tests"
 
 describe("plugins", () => {
   describe("enrichPluginURLs", () => {

package/src/objectStore/objectStore.ts

@@ -3,7 +3,7 @@ import AWS from "aws-sdk"
 import stream from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
-import zlib from "zlib"
+const zlib = require("zlib")
 import { promisify } from "util"
 import { join } from "path"
 import fs from "fs"
@@ -415,7 +415,7 @@ export const downloadTarballDirect = async (
     throw new Error(`unexpected response ${response.statusText}`)
   }
 
-  await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
 }
 
 export const downloadTarball = async (
@@ -431,7 +431,7 @@ export const downloadTarball = async (
   }
 
   const tmpPath = join(budibaseTempDir(), path)
-  await streamPipeline(response.body, zlib.createUnzip(), tar.extract(tmpPath))
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
   if (!env.isTest() && env.SELF_HOSTED) {
     await uploadDirectory(bucketName, tmpPath, path)
   }

package/src/platform/tests/tenants.spec.ts

@@ -1,5 +1,4 @@
-import { structures } from "../../../tests"
-import { DBTestConfiguration } from "../../../tests/extra"
+import { DBTestConfiguration, structures } from "../../../tests"
 import * as tenants from "../tenants"
 
 describe("tenants", () => {

package/src/plugin/utils.ts

@@ -1,12 +1,4 @@
-import {
-  DatasourceFieldType,
-  QueryType,
-  PluginType,
-  AutomationStepType,
-  AutomationStepIdArray,
-  AutomationIOType,
-  AutomationCustomIOType,
-} from "@budibase/types"
+import { DatasourceFieldType, QueryType, PluginType } from "@budibase/types"
 import joi from "joi"
 
 const DATASOURCE_TYPES = [
@@ -27,7 +19,7 @@ function runJoi(validator: joi.Schema, schema: any) {
 
 function validateComponent(schema: any) {
   const validator = joi.object({
-    type: joi.string().allow(PluginType.COMPONENT).required(),
+    type: joi.string().allow("component").required(),
     metadata: joi.object().unknown(true).required(),
     hash: joi.string().optional(),
     version: joi.string().optional(),
@@ -61,7 +53,7 @@ function validateDatasource(schema: any) {
     .required()
 
   const validator = joi.object({
-    type: joi.string().allow(PluginType.DATASOURCE).required(),
+    type: joi.string().allow("datasource").required(),
     metadata: joi.object().unknown(true).required(),
     hash: joi.string().optional(),
     version: joi.string().optional(),
@@ -90,55 +82,6 @@ function validateDatasource(schema: any) {
   runJoi(validator, schema)
 }
 
-function validateAutomation(schema: any) {
-  const basePropsValidator = joi.object().pattern(joi.string(), {
-    type: joi
-      .string()
-      .allow(...Object.values(AutomationIOType))
-      .required(),
-    customType: joi.string().allow(...Object.values(AutomationCustomIOType)),
-    title: joi.string(),
-    description: joi.string(),
-    enum: joi.array().items(joi.string()),
-    pretty: joi.array().items(joi.string()),
-  })
-  const stepSchemaValidator = joi
-    .object({
-      properties: basePropsValidator,
-      required: joi.array().items(joi.string()),
-    })
-    .concat(basePropsValidator)
-    .required()
-  const validator = joi.object({
-    type: joi.string().allow(PluginType.AUTOMATION).required(),
-    metadata: joi.object().unknown(true).required(),
-    hash: joi.string().optional(),
-    version: joi.string().optional(),
-    schema: joi.object({
-      name: joi.string().required(),
-      tagline: joi.string().required(),
-      icon: joi.string().required(),
-      description: joi.string().required(),
-      type: joi
-        .string()
-        .allow(AutomationStepType.ACTION, AutomationStepType.LOGIC)
-        .required(),
-      stepId: joi
-        .string()
-        .disallow(...AutomationStepIdArray)
-        .required(),
-      inputs: joi.object().optional(),
-      schema: joi
-        .object({
-          inputs: stepSchemaValidator,
-          outputs: stepSchemaValidator,
-        })
-        .required(),
-    }),
-  })
-  runJoi(validator, schema)
-}
-
 export function validate(schema: any) {
   switch (schema?.type) {
     case PluginType.COMPONENT:
@@ -147,9 +90,6 @@ export function validate(schema: any) {
     case PluginType.DATASOURCE:
       validateDatasource(schema)
       break
-    case PluginType.AUTOMATION:
-      validateAutomation(schema)
-      break
     default:
       throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`)
   }

package/src/queue/constants.ts

@@ -2,5 +2,4 @@ export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
   AUDIT_LOG = "auditLogQueue",
-  SYSTEM_EVENT_QUEUE = "systemEventQueue",
 }

package/src/security/encryption.ts

@@ -12,7 +12,7 @@ export enum SecretOption {
   ENCRYPTION = "encryption",
 }
 
-export function getSecret(secretOption: SecretOption): string {
+function getSecret(secretOption: SecretOption): string {
   let secret, secretName
   switch (secretOption) {
     case SecretOption.ENCRYPTION:

package/src/security/permissions.ts

@@ -24,7 +24,7 @@ export enum PermissionType {
   QUERY = "query",
 }
 
-export class Permission {
+class Permission {
   type: PermissionType
   level: PermissionLevel
 
@@ -34,7 +34,7 @@ export class Permission {
   }
 }
 
-export function levelToNumber(perm: PermissionLevel) {
+function levelToNumber(perm: PermissionLevel) {
   switch (perm) {
     // not everything has execute privileges
     case PermissionLevel.EXECUTE:
@@ -55,7 +55,7 @@ export function levelToNumber(perm: PermissionLevel) {
  * @param {string} userPermLevel The permission level of the user.
  * @return {string[]} All the permission levels this user is allowed to carry out.
  */
-export function getAllowedLevels(userPermLevel: PermissionLevel): string[] {
+function getAllowedLevels(userPermLevel: PermissionLevel) {
   switch (userPermLevel) {
     case PermissionLevel.EXECUTE:
       return [PermissionLevel.EXECUTE]
@@ -64,9 +64,9 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] {
     case PermissionLevel.WRITE:
     case PermissionLevel.ADMIN:
       return [
-        PermissionLevel.EXECUTE,
         PermissionLevel.READ,
         PermissionLevel.WRITE,
+        PermissionLevel.EXECUTE,
       ]
     default:
       return []
@@ -81,7 +81,7 @@ export enum BuiltinPermissionID {
   POWER = "power",
 }
 
-export const BUILTIN_PERMISSIONS = {
+const BUILTIN_PERMISSIONS = {
   PUBLIC: {
     _id: BuiltinPermissionID.PUBLIC,
     name: "Public",

package/src/tenancy/tenancy.ts

@@ -3,8 +3,8 @@ import {
   getTenantId,
   getTenantIDFromAppID,
   isMultiTenant,
-  getPlatformURL,
 } from "../context"
+import env from "../environment"
 import {
   BBContext,
   TenantResolutionStrategy,
@@ -93,7 +93,7 @@ export const getTenantIDFromCtx = (
   // subdomain
   if (isAllowed(TenantResolutionStrategy.SUBDOMAIN)) {
     // e.g. budibase.app or local.com:10000
-    const platformHost = new URL(getPlatformURL()).host.split(":")[0]
+    const platformHost = new URL(env.PLATFORM_URL).host.split(":")[0]
     // e.g. tenant.budibase.app or tenant.local.com
     const requestHost = ctx.host
     // parse the tenant id from the difference

package/src/users.ts

@@ -8,10 +8,8 @@ import {
   DocumentType,
   SEPARATOR,
   directCouchFind,
-  getGlobalUserParams,
-  pagination,
 } from "./db"
-import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types"
+import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
 import * as context from "./context"
 
@@ -201,41 +199,3 @@ export const searchGlobalUsersByEmail = async (
   }
   return users
 }
-
-const PAGE_LIMIT = 8
-export const paginatedUsers = async ({
-  page,
-  email,
-  appId,
-}: SearchUsersRequest = {}) => {
-  const db = getGlobalDB()
-  // get one extra document, to have the next page
-  const opts: any = {
-    include_docs: true,
-    limit: PAGE_LIMIT + 1,
-  }
-  // add a startkey if the page was specified (anchor)
-  if (page) {
-    opts.startkey = page
-  }
-  // property specifies what to use for the page/anchor
-  let userList: User[],
-    property = "_id",
-    getKey
-  if (appId) {
-    userList = await searchGlobalUsersByApp(appId, opts)
-    getKey = (doc: any) => getGlobalUserByAppPage(appId, doc)
-  } else if (email) {
-    userList = await searchGlobalUsersByEmail(email, opts)
-    property = "email"
-  } else {
-    // no search, query allDocs
-    const response = await db.allDocs(getGlobalUserParams(null, opts))
-    userList = response.rows.map((row: any) => row.doc)
-  }
-  return pagination(userList, PAGE_LIMIT, {
-    paginate: true,
-    property,
-    getKey,
-  })
-}

package/src/utils/hashing.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-export * from "../docIds/newid"
+export * from "../newid"
 const bcrypt = env.JS_BCRYPT ? require("bcryptjs") : require("bcrypt")
 
 const SALT_ROUNDS = env.SALT_ROUNDS || 10

package/src/utils/index.ts

@@ -1,3 +1,2 @@
 export * from "./hashing"
 export * from "./utils"
-export * from "./stringUtils"

package/src/utils/tests/utils.spec.ts

@@ -1,5 +1,4 @@
-import { structures } from "../../../tests"
-import { DBTestConfiguration } from "../../../tests/extra"
+import { structures, DBTestConfiguration } from "../../../tests"
 import * as utils from "../../utils"
 import * as db from "../../db"
 import { Header } from "../../constants"

package/tests/index.ts

@@ -1 +1 @@
-export * from "./core/utilities"
+export * from "./utilities"

package/tests/jestEnv.ts

@@ -3,4 +3,5 @@ process.env.MULTI_TENANCY = "1"
 process.env.NODE_ENV = "jest"
 process.env.MOCK_REDIS = "1"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.REDIS_PASSWORD = "budibase"

package/tests/jestSetup.ts

@@ -1,7 +1,7 @@
-import "./core/logging"
+import "./logging"
 import env from "../src/environment"
 import { cleanup } from "../src/timers"
-import { mocks, testContainerUtils } from "./core/utilities"
+import { mocks, testContainerUtils } from "./utilities"
 
 // must explicitly enable fetch mock
 mocks.fetch.enable()

package/tests/{extra → utilities}/DBTestConfiguration.ts

@@ -1,5 +1,5 @@
-import "../core/utilities/mocks"
-import * as structures from "../core/utilities/structures"
+import "./mocks"
+import * as structures from "./structures"
 import * as testEnv from "./testEnv"
 import * as context from "../../src/context"
 

package/tests/{core/utilities → utilities}/index.ts

@@ -1,6 +1,9 @@
 export * as mocks from "./mocks"
 export * as structures from "./structures"
 export { generator } from "./structures"
+export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
 export * from "./jestUtils"
+
+export { default as DBTestConfiguration } from "./DBTestConfiguration"