@budibase/backend-core 3.2.4 → 3.2.6

package/src/logging/pino/logger.ts

@@ -1,239 +0,0 @@
-import pino, { LoggerOptions } from "pino"
-import pinoPretty from "pino-pretty"
-
-import { IdentityType } from "@budibase/types"
-import env from "../../environment"
-import * as context from "../../context"
-import * as correlation from "../correlation"
-import tracer from "dd-trace"
-import { formats } from "dd-trace/ext"
-
-import { localFileDestination } from "../system"
-
-function isPlainObject(obj: any) {
-  return typeof obj === "object" && obj !== null && !(obj instanceof Error)
-}
-
-function isError(obj: any) {
-  return obj instanceof Error
-}
-
-function isMessage(obj: any) {
-  return typeof obj === "string"
-}
-
-// LOGGER
-
-let pinoInstance: pino.Logger | undefined
-if (!env.DISABLE_PINO_LOGGER) {
-  const level = env.LOG_LEVEL
-  const pinoOptions: LoggerOptions = {
-    level,
-    formatters: {
-      level: level => {
-        return { level: level.toUpperCase() }
-      },
-      bindings: () => {
-        if (env.SELF_HOSTED) {
-          // "service" is being injected in datadog using the pod names,
-          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
-          return {
-            service: env.SERVICE_NAME,
-          }
-        } else {
-          return {}
-        }
-      },
-    },
-    timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
-  }
-
-  const destinations: pino.StreamEntry[] = []
-
-  destinations.push(
-    env.isDev()
-      ? {
-          stream: pinoPretty({ singleLine: true }),
-          level: level as pino.Level,
-        }
-      : { stream: process.stdout, level: level as pino.Level }
-  )
-
-  if (env.SELF_HOSTED) {
-    destinations.push({
-      stream: localFileDestination(),
-      level: level as pino.Level,
-    })
-  }
-
-  pinoInstance = destinations.length
-    ? pino(pinoOptions, pino.multistream(destinations))
-    : pino(pinoOptions)
-
-  // CONSOLE OVERRIDES
-
-  interface MergingObject {
-    objects?: any[]
-    tenantId?: string
-    appId?: string
-    automationId?: string
-    identityId?: string
-    identityType?: IdentityType
-    correlationId?: string
-    err?: Error
-  }
-
-  /**
-   * Backwards compatibility between console logging statements
-   * and pino logging requirements.
-   */
-  const getLogParams = (args: any[]): [MergingObject, string] => {
-    let error = undefined
-    let objects: any[] = []
-    let message = ""
-
-    args.forEach(arg => {
-      if (isMessage(arg)) {
-        message = `${message} ${arg}`.trimStart()
-      }
-      if (isPlainObject(arg)) {
-        objects.push(arg)
-      }
-      if (isError(arg)) {
-        error = arg
-      }
-    })
-
-    const identity = getIdentity()
-
-    let contextObject = {}
-
-    contextObject = {
-      tenantId: getTenantId(),
-      appId: getAppId(),
-      automationId: getAutomationId(),
-      identityId: identity?._id,
-      identityType: identity?.type,
-      correlationId: correlation.getId(),
-    }
-
-    const span = tracer.scope().active()
-    if (span) {
-      tracer.inject(span.context(), formats.LOG, contextObject)
-    }
-
-    const mergingObject: any = {
-      err: error,
-      pid: process.pid,
-      ...contextObject,
-    }
-
-    if (objects.length) {
-      // init generic data object for params supplied that don't have a
-      // '_logKey' field. This prints an object using argument index as the key
-      // e.g. { 0: {}, 1: {} }
-      const data: any = {}
-      let dataIndex = 0
-
-      for (let i = 0; i < objects.length; i++) {
-        const object = objects[i]
-        // the object has specified a log key
-        // use this instead of generic key
-        const logKey = object._logKey
-        if (logKey) {
-          delete object._logKey
-          mergingObject[logKey] = object
-        } else {
-          data[dataIndex] = object
-          dataIndex++
-        }
-      }
-
-      if (Object.keys(data).length) {
-        mergingObject.data = data
-      }
-    }
-
-    return [mergingObject, message]
-  }
-
-  console.log = (...arg: any[]) => {
-    const [obj, msg] = getLogParams(arg)
-    pinoInstance?.info(obj, msg)
-  }
-  console.info = (...arg: any[]) => {
-    const [obj, msg] = getLogParams(arg)
-    pinoInstance?.info(obj, msg)
-  }
-  console.warn = (...arg: any[]) => {
-    const [obj, msg] = getLogParams(arg)
-    pinoInstance?.warn(obj, msg)
-  }
-  console.error = (...arg: any[]) => {
-    const [obj, msg] = getLogParams(arg)
-    pinoInstance?.error(obj, msg)
-  }
-
-  /**
-   * custom trace impl - this resembles the node trace behaviour rather
-   * than traditional trace logging
-   * @param arg
-   */
-  console.trace = (...arg: any[]) => {
-    const [obj, msg] = getLogParams(arg)
-    if (!obj.err) {
-      // to get stack trace
-      obj.err = new Error()
-    }
-    pinoInstance?.trace(obj, msg)
-  }
-
-  console.debug = (...arg: any) => {
-    const [obj, msg] = getLogParams(arg)
-    pinoInstance?.debug(obj, msg)
-  }
-
-  // CONTEXT
-
-  const getTenantId = () => {
-    let tenantId
-    try {
-      tenantId = context.getTenantId()
-    } catch (e: any) {
-      // do nothing
-    }
-    return tenantId
-  }
-
-  const getAppId = () => {
-    let appId
-    try {
-      appId = context.getAppId()
-    } catch (e) {
-      // do nothing
-    }
-    return appId
-  }
-
-  const getAutomationId = () => {
-    let appId
-    try {
-      appId = context.getAutomationId()
-    } catch (e) {
-      // do nothing
-    }
-    return appId
-  }
-
-  const getIdentity = () => {
-    let identity
-    try {
-      identity = context.getIdentity()
-    } catch (e) {
-      // do nothing
-    }
-    return identity
-  }
-}
-
-export const logger = pinoInstance

package/src/logging/pino/middleware.ts

@@ -1,48 +0,0 @@
-import env from "../../environment"
-import { logger } from "./logger"
-import { IncomingMessage } from "http"
-
-const pino = require("koa-pino-logger")
-
-import { Options } from "pino-http"
-import { Ctx } from "@budibase/types"
-
-const correlator = require("correlation-id")
-
-export function pinoSettings(): Options {
-  return {
-    logger,
-    genReqId: correlator.getId,
-    autoLogging: {
-      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
-    },
-    serializers: {
-      req: req => {
-        return {
-          method: req.method,
-          url: req.url,
-          correlationId: req.id,
-        }
-      },
-      res: res => {
-        return {
-          status: res.statusCode,
-        }
-      },
-    },
-  }
-}
-
-function getMiddleware() {
-  if (env.HTTP_LOGGING) {
-    return pino(pinoSettings())
-  } else {
-    return (ctx: Ctx, next: any) => {
-      return next()
-    }
-  }
-}
-
-const pinoMiddleware = getMiddleware()
-
-export default pinoMiddleware

package/src/logging/system.ts

@@ -1,81 +0,0 @@
-import fs from "fs"
-import path from "path"
-import * as rfs from "rotating-file-stream"
-
-import env from "../environment"
-import { budibaseTempDir } from "../objectStore"
-
-const logsFileName = `budibase.log`
-const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
-
-const logsPath = path.join(budibaseTempDir(), "systemlogs")
-
-function getFullPath(fileName: string) {
-  return path.join(logsPath, fileName)
-}
-
-export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
-  const regex = /(\d+)([A-Za-z])/
-  const match = totalMaxSize?.match(regex)
-  if (!match) {
-    console.warn(`totalMaxSize does not have a valid value`, {
-      totalMaxSize,
-    })
-    return undefined
-  }
-
-  const size = +match[1]
-  const unit = match[2]
-  if (size === 1) {
-    switch (unit) {
-      case "B":
-        return { size: `${size}B`, totalHistoryFiles: 1 }
-      case "K":
-        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
-      case "M":
-        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
-      case "G":
-        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
-      default:
-        return undefined
-    }
-  }
-
-  if (size % 2 === 0) {
-    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
-  }
-
-  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
-}
-
-export function localFileDestination() {
-  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
-  const outFile = rfs.createStream(logsFileName, {
-    // As we have a rolling size, we want to half the max size
-    size: fileInfo?.size,
-    path: logsPath,
-    maxFiles: fileInfo?.totalHistoryFiles || 1,
-    immutable: true,
-    history: budibaseLogsHistoryFileName,
-    initialRotation: false,
-  })
-
-  return outFile
-}
-
-export function getLogReadStream() {
-  const streams = []
-  const historyFile = getFullPath(budibaseLogsHistoryFileName)
-  if (fs.existsSync(historyFile)) {
-    const fileContent = fs.readFileSync(historyFile, "utf-8")
-    const historyFiles = fileContent.split("\n")
-    for (const historyFile of historyFiles.filter(x => x)) {
-      streams.push(fs.readFileSync(historyFile))
-    }
-  }
-
-  streams.push(fs.readFileSync(getFullPath(logsFileName)))
-
-  const combinedContent = Buffer.concat(streams)
-  return combinedContent
-}

package/src/logging/tests/system.spec.ts

@@ -1,61 +0,0 @@
-import { getSingleFileMaxSizeInfo } from "../system"
-
-describe("system", () => {
-  describe("getSingleFileMaxSizeInfo", () => {
-    it.each([
-      ["100B", "50B"],
-      ["200K", "100K"],
-      ["20M", "10M"],
-      ["4G", "2G"],
-    ])(
-      "Halving even number (%s) returns halved size and 1 history file (%s)",
-      (totalValue, expectedMaxSize) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles: 1,
-        })
-      }
-    )
-
-    it.each([
-      ["5B", "1B", 4],
-      ["17K", "1K", 16],
-      ["21M", "1M", 20],
-      ["3G", "1G", 2],
-    ])(
-      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
-      (totalValue, expectedMaxSize, totalHistoryFiles) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles,
-        })
-      }
-    )
-
-    it.each([
-      ["1B", "1B"],
-      ["1K", "500B"],
-      ["1M", "500K"],
-      ["1G", "500M"],
-    ])(
-      "Halving '%s' returns halved unit (%s)",
-      (totalValue, expectedMaxSize) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles: 1,
-        })
-      }
-    )
-
-    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
-      "Halving wrongly formatted value ('%s') returns undefined",
-      totalValue => {
-        const result = getSingleFileMaxSizeInfo(totalValue!)
-        expect(result).toBeUndefined()
-      }
-    )
-  })
-})

package/src/middleware/adminOnly.ts

@@ -1,9 +0,0 @@
-import { UserCtx } from "@budibase/types"
-import { isAdmin } from "../users"
-
-export default async (ctx: UserCtx, next: any) => {
-  if (!ctx.internal && !isAdmin(ctx.user)) {
-    ctx.throw(403, "Admin user only endpoint.")
-  }
-  return next()
-}

package/src/middleware/auditLog.ts

@@ -1,6 +0,0 @@
-import { BBContext } from "@budibase/types"
-
-export default async (ctx: BBContext | any, next: any) => {
-  // Placeholder for audit log middleware
-  return next()
-}

package/src/middleware/authenticated.ts

@@ -1,247 +0,0 @@
-import { Cookie, Header } from "../constants"
-import {
-  clearCookie,
-  getCookie,
-  isValidInternalAPIKey,
-  openJwt,
-} from "../utils"
-import { getUser } from "../cache/user"
-import { getSession, updateSessionTTL } from "../security/sessions"
-import { buildMatcherRegex, matches } from "./matchers"
-import { queryGlobalView, SEPARATOR, ViewName } from "../db"
-import { doInTenant, getGlobalDB } from "../context"
-import { decrypt } from "../security/encryption"
-import * as identity from "../context/identity"
-import env from "../environment"
-import {
-  Ctx,
-  EndpointMatcher,
-  LoginMethod,
-  SessionCookie,
-  User,
-} from "@budibase/types"
-import { ErrorCode, InvalidAPIKeyError } from "../errors"
-import tracer from "dd-trace"
-
-const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
-  ? parseInt(env.SESSION_UPDATE_PERIOD)
-  : 60 * 1000
-
-interface FinaliseOpts {
-  authenticated?: boolean
-  internal?: boolean
-  publicEndpoint?: boolean
-  version?: string
-  user?: User | { tenantId: string }
-  loginMethod?: LoginMethod
-}
-
-function timeMinusOneMinute() {
-  return new Date(Date.now() - ONE_MINUTE).toISOString()
-}
-
-function finalise(ctx: Ctx, opts: FinaliseOpts = {}) {
-  ctx.publicEndpoint = opts.publicEndpoint || false
-  ctx.isAuthenticated = opts.authenticated || false
-  ctx.loginMethod = opts.loginMethod
-  ctx.user = opts.user
-  ctx.internal = opts.internal || false
-  ctx.version = opts.version
-}
-
-async function checkApiKey(
-  apiKey: string,
-  populateUser?: (
-    userId: string,
-    tenantId: string,
-    email?: string
-  ) => Promise<User>
-) {
-  // check both the primary and the fallback internal api keys
-  // this allows for rotation
-  if (isValidInternalAPIKey(apiKey)) {
-    return { valid: true, user: undefined }
-  }
-  const decrypted = decrypt(apiKey)
-  const tenantId = decrypted.split(SEPARATOR)[0]
-  return doInTenant(tenantId, async () => {
-    let userId
-    try {
-      const db = getGlobalDB()
-      // api key is encrypted in the database
-      userId = (await queryGlobalView(
-        ViewName.BY_API_KEY,
-        {
-          key: apiKey,
-        },
-        db
-      )) as string
-    } catch (err) {
-      userId = undefined
-    }
-    if (userId) {
-      return {
-        valid: true,
-        user: await getUser({
-          userId,
-          tenantId,
-          populateUser,
-        }),
-      }
-    } else {
-      throw new InvalidAPIKeyError()
-    }
-  })
-}
-
-/**
- * This middleware is tenancy aware, so that it does not depend on other middlewares being used.
- * The tenancy modules should not be used here and it should be assumed that the tenancy context
- * has not yet been populated.
- */
-export default function (
-  noAuthPatterns: EndpointMatcher[] = [],
-  opts: { publicAllowed?: boolean; populateUser?: Function } = {
-    publicAllowed: false,
-  }
-) {
-  const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
-  return async (ctx: Ctx | any, next: any) => {
-    let publicEndpoint = false
-    const version = ctx.request.headers[Header.API_VER]
-    // the path is not authenticated
-    const found = matches(ctx, noAuthOptions)
-    if (found) {
-      publicEndpoint = true
-    }
-    try {
-      // check the actual user is authenticated first, try header or cookie
-      let headerToken = ctx.request.headers[Header.TOKEN]
-
-      const authCookie =
-        getCookie<SessionCookie>(ctx, Cookie.Auth) ||
-        openJwt<SessionCookie>(headerToken)
-      let apiKey = ctx.request.headers[Header.API_KEY]
-
-      if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {
-        apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1]
-      }
-
-      const tenantId = ctx.request.headers[Header.TENANT_ID]
-      let authenticated: boolean = false,
-        user: User | { tenantId: string } | undefined = undefined,
-        internal: boolean = false,
-        loginMethod: LoginMethod | undefined = undefined
-      if (authCookie && !apiKey) {
-        const sessionId = authCookie.sessionId
-        const userId = authCookie.userId
-        let session
-        try {
-          // getting session handles error checking (if session exists etc)
-          session = await getSession(userId, sessionId)
-          if (opts && opts.populateUser) {
-            user = await getUser({
-              userId,
-              tenantId: session.tenantId,
-              email: session.email,
-              populateUser: opts.populateUser(ctx),
-            })
-          } else {
-            user = await getUser({
-              userId,
-              tenantId: session.tenantId,
-              email: session.email,
-            })
-          }
-          // @ts-ignore
-          user.csrfToken = session.csrfToken
-          loginMethod = LoginMethod.COOKIE
-
-          if (session?.lastAccessedAt < timeMinusOneMinute()) {
-            // make sure we denote that the session is still in use
-            await updateSessionTTL(session)
-          }
-          authenticated = true
-        } catch (err: any) {
-          authenticated = false
-          console.error(`Auth Error: ${err.message}`)
-          // remove the cookie as the user does not exist anymore
-          clearCookie(ctx, Cookie.Auth)
-        }
-      }
-      // this is an internal request, no user made it
-      if (!authenticated && apiKey) {
-        const populateUser: (
-          userId: string,
-          tenantId: string,
-          email?: string
-        ) => Promise<User> = opts.populateUser ? opts.populateUser(ctx) : null
-        const { valid, user: foundUser } = await checkApiKey(
-          apiKey,
-          populateUser
-        )
-        if (valid) {
-          authenticated = true
-          loginMethod = LoginMethod.API_KEY
-          user = foundUser
-          internal = !foundUser
-        }
-      }
-      if (!user && tenantId) {
-        user = { tenantId }
-      } else if (user && "password" in user) {
-        delete user.password
-      }
-      // be explicit
-      if (!authenticated) {
-        authenticated = false
-      }
-
-      const isUser = (
-        user: any
-      ): user is User & { budibaseAccess?: string } => {
-        return user && user.email
-      }
-
-      if (isUser(user)) {
-        tracer.setUser({
-          id: user._id!,
-          tenantId: user.tenantId,
-          budibaseAccess: user.budibaseAccess,
-          status: user.status,
-        })
-      }
-
-      // isAuthenticated is a function, so use a variable to be able to check authed state
-      finalise(ctx, {
-        authenticated,
-        user,
-        internal,
-        version,
-        publicEndpoint,
-        loginMethod,
-      })
-
-      if (isUser(user)) {
-        return identity.doInUserContext(user, ctx, next)
-      } else {
-        return next()
-      }
-    } catch (err: any) {
-      console.error(`Auth Error: ${err.message}`)
-      // invalid token, clear the cookie
-      if (err?.name === "JsonWebTokenError") {
-        clearCookie(ctx, Cookie.Auth)
-      } else if (err?.code === ErrorCode.INVALID_API_KEY) {
-        ctx.throw(403, err.message)
-      }
-      // allow configuring for public access
-      if ((opts && opts.publicAllowed) || publicEndpoint) {
-        finalise(ctx, { authenticated: false, version, publicEndpoint })
-        return next()
-      } else {
-        ctx.throw(err.status || 403, err)
-      }
-    }
-  }
-}