@budibase/backend-core 2.5.8 → 2.5.10-alpha.0

package/src/events/publishers/license.ts

@@ -3,7 +3,6 @@ import {
   Event,
   LicenseActivatedEvent,
   LicensePlanChangedEvent,
-  LicenseTierChangedEvent,
   PlanType,
   Account,
   LicensePortalOpenedEvent,
@@ -11,22 +10,23 @@ import {
   LicenseCheckoutOpenedEvent,
   LicensePaymentFailedEvent,
   LicensePaymentRecoveredEvent,
+  PriceDuration,
 } from "@budibase/types"
 
-async function tierChanged(account: Account, from: number, to: number) {
-  const properties: LicenseTierChangedEvent = {
-    accountId: account.accountId,
-    to,
-    from,
+async function planChanged(
+  account: Account,
+  opts: {
+    from: PlanType
+    to: PlanType
+    fromQuantity: number | undefined
+    toQuantity: number | undefined
+    fromDuration: PriceDuration | undefined
+    toDuration: PriceDuration | undefined
   }
-  await publishEvent(Event.LICENSE_TIER_CHANGED, properties)
-}
-
-async function planChanged(account: Account, from: PlanType, to: PlanType) {
+) {
   const properties: LicensePlanChangedEvent = {
     accountId: account.accountId,
-    to,
-    from,
+    ...opts,
   }
   await publishEvent(Event.LICENSE_PLAN_CHANGED, properties)
 }
@@ -74,7 +74,6 @@ async function paymentRecovered(account: Account) {
 }
 
 export default {
-  tierChanged,
   planChanged,
   activated,
   checkoutOpened,

package/src/events/publishers/user.ts

@@ -15,10 +15,12 @@ import {
   UserUpdatedEvent,
   UserOnboardingEvent,
 } from "@budibase/types"
+import { isScim } from "../../context"
 
 async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
+    viaScim: isScim(),
     audited: {
       email: user.email,
     },
@@ -29,6 +31,7 @@ async function created(user: User, timestamp?: number) {
 async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
+    viaScim: isScim(),
     audited: {
       email: user.email,
     },
@@ -39,6 +42,7 @@ async function updated(user: User) {
 async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
+    viaScim: isScim(),
     audited: {
       email: user.email,
     },

package/src/index.ts

@@ -27,6 +27,7 @@ export * as errors from "./errors"
 export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
+export * as docUpdates from "./docUpdates"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

package/src/installation.ts

@@ -6,8 +6,7 @@ import { Installation, IdentityType, Database } from "@budibase/types"
 import * as context from "./context"
 import semver from "semver"
 import { bustCache, withCache, TTL, CacheKey } from "./cache/generic"
-
-const pkg = require("../package.json")
+import environment from "./environment"
 
 export const getInstall = async (): Promise<Installation> => {
   return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
@@ -18,7 +17,7 @@ async function createInstallDoc(platformDb: Database) {
   const install: Installation = {
     _id: StaticDatabases.PLATFORM_INFO.docs.install,
     installId: newid(),
-    version: pkg.version,
+    version: environment.VERSION,
   }
   try {
     const resp = await platformDb.put(install)
@@ -33,7 +32,7 @@ async function createInstallDoc(platformDb: Database) {
   }
 }
 
-const getInstallFromDB = async (): Promise<Installation> => {
+export const getInstallFromDB = async (): Promise<Installation> => {
   return doWithDB(
     StaticDatabases.PLATFORM_INFO.name,
     async (platformDb: any) => {
@@ -80,7 +79,7 @@ export const checkInstallVersion = async (): Promise<void> => {
   const install = await getInstall()
 
   const currentVersion = install.version
-  const newVersion = pkg.version
+  const newVersion = environment.VERSION
 
   if (currentVersion !== newVersion) {
     const isUpgrade = semver.gt(newVersion, currentVersion)

package/src/logging/alerts.ts

@@ -0,0 +1,26 @@
+const NonErrors = ["AccountError"]
+
+function isSuppressed(e?: any) {
+  return e && e["suppressAlert"]
+}
+
+export function logAlert(message: string, e?: any) {
+  if (e && NonErrors.includes(e.name) && isSuppressed(e)) {
+    return
+  }
+  console.error(`bb-alert: ${message}`, e)
+}
+
+export function logAlertWithInfo(
+  message: string,
+  db: string,
+  id: string,
+  error: any
+) {
+  message = `${message} - db: ${db} - doc: ${id} - error: `
+  logAlert(message, error)
+}
+
+export function logWarn(message: string) {
+  console.warn(`bb-warn: ${message}`)
+}

package/src/logging/correlation/correlation.ts

@@ -0,0 +1,13 @@
+import { Header } from "../../constants"
+const correlator = require("correlation-id")
+
+export const setHeader = (headers: any) => {
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    headers[Header.CORRELATION_ID] = correlationId
+  }
+}
+
+export function getId() {
+  return correlator.getId()
+}

package/src/logging/correlation/index.ts

@@ -0,0 +1 @@
+export * from "./correlation"

package/src/logging/correlation/middleware.ts

@@ -0,0 +1,17 @@
+import { Header } from "../../constants"
+import { v4 as uuid } from "uuid"
+const correlator = require("correlation-id")
+
+const correlation = (ctx: any, next: any) => {
+  // use the provided correlation id header if present
+  let correlationId = ctx.headers[Header.CORRELATION_ID]
+  if (!correlationId) {
+    correlationId = uuid()
+  }
+
+  return correlator.withId(correlationId, () => {
+    return next()
+  })
+}
+
+export default correlation

package/src/logging/index.ts

@@ -0,0 +1,6 @@
+export * as correlation from "./correlation/correlation"
+export { logger } from "./pino/logger"
+export * from "./alerts"
+
+// turn off or on context logging i.e. tenantId, appId etc
+export let LOG_CONTEXT = true

package/src/logging/pino/logger.ts

@@ -0,0 +1,173 @@
+import env from "../../environment"
+import pino, { LoggerOptions } from "pino"
+import * as context from "../../context"
+import * as correlation from "../correlation"
+import { IdentityType } from "@budibase/types"
+import { LOG_CONTEXT } from "../index"
+
+// LOGGER
+
+let pinoInstance: pino.Logger | undefined
+if (!env.DISABLE_PINO_LOGGER) {
+  const pinoOptions: LoggerOptions = {
+    level: env.LOG_LEVEL,
+    formatters: {
+      level: label => {
+        return { level: label.toUpperCase() }
+      },
+      bindings: () => {
+        return {}
+      },
+    },
+    timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
+  }
+
+  if (env.isDev()) {
+    pinoOptions.transport = {
+      target: "pino-pretty",
+      options: {
+        singleLine: true,
+      },
+    }
+  }
+
+  pinoInstance = pino(pinoOptions)
+
+  // CONSOLE OVERRIDES
+
+  interface MergingObject {
+    objects?: any[]
+    tenantId?: string
+    appId?: string
+    identityId?: string
+    identityType?: IdentityType
+    correlationId?: string
+    err?: Error
+  }
+
+  function isPlainObject(obj: any) {
+    return typeof obj === "object" && obj !== null && !(obj instanceof Error)
+  }
+
+  function isError(obj: any) {
+    return obj instanceof Error
+  }
+
+  function isMessage(obj: any) {
+    return typeof obj === "string"
+  }
+
+  /**
+   * Backwards compatibility between console logging statements
+   * and pino logging requirements.
+   */
+  function getLogParams(args: any[]): [MergingObject, string] {
+    let error = undefined
+    let objects: any[] = []
+    let message = ""
+
+    args.forEach(arg => {
+      if (isMessage(arg)) {
+        message = `${message} ${arg}`.trimStart()
+      }
+      if (isPlainObject(arg)) {
+        objects.push(arg)
+      }
+      if (isError(arg)) {
+        error = arg
+      }
+    })
+
+    const identity = getIdentity()
+
+    let contextObject = {}
+
+    if (LOG_CONTEXT) {
+      contextObject = {
+        tenantId: getTenantId(),
+        appId: getAppId(),
+        identityId: identity?._id,
+        identityType: identity?.type,
+        correlationId: correlation.getId(),
+      }
+    }
+
+    const mergingObject = {
+      objects: objects.length ? objects : undefined,
+      err: error,
+      ...contextObject,
+    }
+
+    return [mergingObject, message]
+  }
+
+  console.log = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.info = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.warn = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.warn(obj, msg)
+  }
+  console.error = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.error(obj, msg)
+  }
+
+  /**
+   * custom trace impl - this resembles the node trace behaviour rather
+   * than traditional trace logging
+   * @param arg
+   */
+  console.trace = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    if (!obj.err) {
+      // to get stack trace
+      obj.err = new Error()
+    }
+    pinoInstance?.trace(obj, msg)
+  }
+
+  console.debug = (...arg: any) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.debug(obj, msg)
+  }
+
+  // CONTEXT
+
+  const getTenantId = () => {
+    let tenantId
+    try {
+      tenantId = context.getTenantId()
+    } catch (e: any) {
+      // do nothing
+    }
+    return tenantId
+  }
+
+  const getAppId = () => {
+    let appId
+    try {
+      appId = context.getAppId()
+    } catch (e) {
+      // do nothing
+    }
+    return appId
+  }
+
+  const getIdentity = () => {
+    let identity
+    try {
+      identity = context.getIdentity()
+    } catch (e) {
+      // do nothing
+    }
+    return identity
+  }
+}
+
+export const logger = pinoInstance

package/src/logging/pino/middleware.ts

@@ -0,0 +1,45 @@
+import env from "../../environment"
+import { logger } from "./logger"
+import { IncomingMessage } from "http"
+const pino = require("koa-pino-logger")
+import { Options } from "pino-http"
+import { Ctx } from "@budibase/types"
+const correlator = require("correlation-id")
+
+export function pinoSettings(): Options {
+  return {
+    logger,
+    genReqId: correlator.getId,
+    autoLogging: {
+      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
+    },
+    serializers: {
+      req: req => {
+        return {
+          method: req.method,
+          url: req.url,
+          correlationId: req.id,
+        }
+      },
+      res: res => {
+        return {
+          status: res.statusCode,
+        }
+      },
+    },
+  }
+}
+
+function getMiddleware() {
+  if (env.HTTP_LOGGING) {
+    return pino(pinoSettings())
+  } else {
+    return (ctx: Ctx, next: any) => {
+      return next()
+    }
+  }
+}
+
+const pinoMiddleware = getMiddleware()
+
+export default pinoMiddleware

package/src/middleware/authenticated.ts

@@ -44,7 +44,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
   // check both the primary and the fallback internal api keys
   // this allows for rotation
   if (isValidInternalAPIKey(apiKey)) {
-    return { valid: true }
+    return { valid: true, user: undefined }
   }
   const decrypted = decrypt(apiKey)
   const tenantId = decrypted.split(SEPARATOR)[0]
@@ -96,9 +96,15 @@ export default function (
     }
     try {
       // check the actual user is authenticated first, try header or cookie
-      const headerToken = ctx.request.headers[Header.TOKEN]
+      let headerToken = ctx.request.headers[Header.TOKEN]
+
       const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
-      const apiKey = ctx.request.headers[Header.API_KEY]
+      let apiKey = ctx.request.headers[Header.API_KEY]
+
+      if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {
+        apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1]
+      }
+
       const tenantId = ctx.request.headers[Header.TENANT_ID]
       let authenticated = false,
         user = null,

package/src/middleware/errorHandling.ts

@@ -1,6 +1,5 @@
 import { APIError } from "@budibase/types"
 import * as errors from "../errors"
-import env from "../environment"
 
 export async function errorHandling(ctx: any, next: any) {
   try {
@@ -9,9 +8,10 @@ export async function errorHandling(ctx: any, next: any) {
     const status = err.status || err.statusCode || 500
     ctx.status = status
 
-    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
-      ctx.log.error(err)
-      console.trace(err)
+    if (status >= 400 && status < 500) {
+      console.warn(err)
+    } else {
+      console.error(err)
     }
 
     const error = errors.getPublicError(err)

package/src/middleware/index.ts

@@ -14,7 +14,8 @@ export { default as csrf } from "./csrf"
 export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
-export { default as logging } from "./logging"
+export { default as pino } from "../logging/pino/middleware"
+export { default as correlation } from "../logging/correlation/middleware"
 export { default as errorHandling } from "./errorHandling"
 export { default as querystringToBody } from "./querystringToBody"
 export * as joiValidator from "./joi-validator"

package/src/middleware/passport/sso/oidc.ts

@@ -1,6 +1,7 @@
 import fetch from "node-fetch"
 import * as sso from "./sso"
 import { ssoCallbackUrl } from "../utils"
+import { validEmail } from "../../../utils"
 import {
   ConfigType,
   OIDCInnerConfig,
@@ -11,6 +12,7 @@ import {
   JwtClaims,
   SaveSSOUserFunction,
 } from "@budibase/types"
+
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
 export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
@@ -86,15 +88,6 @@ function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) {
   )
 }
 
-function validEmail(value: string) {
-  return (
-    value &&
-    !!value.match(
-      /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
-    )
-  )
-}
-
 /**
  * Create an instance of the oidc passport strategy. This wrapper fetches the configuration
  * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.

package/src/middleware/passport/sso/tests/sso.spec.ts

@@ -1,4 +1,5 @@
-import { structures, testEnv, mocks } from "../../../../../tests"
+import { structures, mocks } from "../../../../../tests"
+import { testEnv } from "../../../../../tests/extra"
 import { SSOAuthDetails, User } from "@budibase/types"
 
 import { HTTPError } from "../../../../errors"

package/src/migrations/migrations.ts

@@ -99,9 +99,7 @@ export const runMigration = async (
           options.force[migrationType] &&
           options.force[migrationType].includes(migrationName)
         ) {
-          log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-          )
+          log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
         } else {
           // no force, exit
           return
@@ -111,7 +109,7 @@ export const runMigration = async (
       // check if the migration is not a no-op
       if (!options.noOp) {
         log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+          `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
         )
 
         if (migration.preventRetry) {
@@ -131,9 +129,7 @@ export const runMigration = async (
           await migration.fn(db)
         }
 
-        log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
-        )
+        log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
       }
 
       // mark as complete
@@ -141,7 +137,7 @@ export const runMigration = async (
       await db.put(doc)
     } catch (err) {
       console.error(
-        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
         err
       )
       throw err

package/src/migrations/tests/__snapshots__/migrations.spec.ts.snap

@@ -1,7 +1,7 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`migrations should match snapshot 1`] = `
-Object {
+{
   "_id": "migrations",
   "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
   "createdAt": "2020-01-01T00:00:00.000Z",

package/src/migrations/tests/migrations.spec.ts

@@ -1,4 +1,4 @@
-import { testEnv, DBTestConfiguration } from "../../../tests"
+import { testEnv, DBTestConfiguration } from "../../../tests/extra"
 import * as migrations from "../index"
 import * as context from "../../context"
 import { MigrationType } from "@budibase/types"

package/src/objectStore/buckets/tests/app.spec.ts

@@ -1,6 +1,6 @@
 import * as app from "../app"
 import { getAppFileUrl } from "../app"
-import { testEnv } from "../../../../tests"
+import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
   beforeEach(() => {

package/src/objectStore/buckets/tests/global.spec.ts

@@ -1,5 +1,5 @@
 import * as global from "../global"
-import { testEnv } from "../../../../tests"
+import { testEnv } from "../../../../tests/extra"
 
 describe("global", () => {
   describe("getGlobalFileUrl", () => {

package/src/objectStore/buckets/tests/plugins.spec.ts

@@ -1,5 +1,6 @@
 import * as plugins from "../plugins"
-import { structures, testEnv } from "../../../../tests"
+import { structures } from "../../../../tests"
+import { testEnv } from "../../../../tests/extra"
 
 describe("plugins", () => {
   describe("enrichPluginURLs", () => {

package/src/objectStore/objectStore.ts

@@ -3,7 +3,7 @@ import AWS from "aws-sdk"
 import stream from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
-const zlib = require("zlib")
+import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
 import fs from "fs"
@@ -415,7 +415,7 @@ export const downloadTarballDirect = async (
     throw new Error(`unexpected response ${response.statusText}`)
   }
 
-  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+  await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
 export const downloadTarball = async (
@@ -431,7 +431,7 @@ export const downloadTarball = async (
   }
 
   const tmpPath = join(budibaseTempDir(), path)
-  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
+  await streamPipeline(response.body, zlib.createUnzip(), tar.extract(tmpPath))
   if (!env.isTest() && env.SELF_HOSTED) {
     await uploadDirectory(bucketName, tmpPath, path)
   }

package/src/platform/tests/tenants.spec.ts

@@ -1,4 +1,5 @@
-import { DBTestConfiguration, structures } from "../../../tests"
+import { structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
 import * as tenants from "../tenants"
 
 describe("tenants", () => {